rsshub 1.0.0-master.f8b1c8d → 1.0.0-master.f961961

package/lib/v2/radio/album.js

@@ -0,0 +1,106 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+const CryptoJS = require('crypto-js');
+
+module.exports = async (ctx) => {
+    const KEY = 'f0fc4c668392f9f9a447e48584c214ee';
+
+    const id = ctx.params.id;
+    const size = ctx.query.limit ? parseInt(ctx.query.limit) : 100;
+
+    const rootUrl = 'https://www.radio.cn';
+    const apiRootUrl = 'https://ytmsout.radio.cn';
+    const detailRootUrl = 'https://ytapi.radio.cn';
+
+    const iconUrl = `${rootUrl}/pc-portal/image/icon_32.jpg`;
+    const currentUrl = `${rootUrl}/pc-portal/sanji/detail.html?columnId=${id}`;
+    const detailUrl = `${detailRootUrl}/ytsrv/srv/wifimusicbox/demand/detail`;
+
+    const detailResponse = await got({
+        method: 'post',
+        url: detailUrl,
+        form: {
+            pageIndex: '0',
+            sortType: '2',
+            mobileId: '',
+            providerCode: '25010',
+            pid: id,
+            paySongFlag: '1',
+            richText: '1',
+            h5flag: '1',
+        },
+        headers: {
+            'Content-Type': 'application/x-www-form-urlencoded',
+            version: '4.0.0',
+            providerCode: '25010',
+            equipmentSource: 'WEB',
+        },
+    });
+
+    const data = detailResponse.data;
+
+    const count = data.count;
+
+    let pageNo = 1,
+        pageSize = count - size;
+
+    if (pageSize <= 0) {
+        pageNo = 0;
+        pageSize = count;
+    }
+
+    const params = `albumId=${id}&pageNo=${pageNo}&pageSize=${pageSize}`;
+    const apiUrl = `${apiRootUrl}/web/appSingle/pageByAlbum?${params}`;
+
+    const timestamp = new Date().getTime();
+    const sign = CryptoJS.MD5(`${params}&timestamp=${timestamp}&key=${KEY}`).toString().toUpperCase();
+
+    const response = await got({
+        method: 'get',
+        url: apiUrl,
+        headers: {
+            sign,
+            timestamp,
+            'Content-Type': 'application/json',
+            equipmentId: '0000',
+            platformCode: 'WEB',
+        },
+    });
+
+    const items = response.data.data.data.map((item) => {
+        let enclosure_url = item.playUrlHigh ?? item.playUrlLow;
+        enclosure_url = /\.m3u8$/.test(enclosure_url) ? item.downloadUrl : enclosure_url;
+
+        const enclosure_type = `audio/${enclosure_url.match(/\.(\w+)$/)[1]}`;
+
+        return {
+            guid: item.id,
+            title: item.name,
+            link: enclosure_url,
+            description: art(path.join(__dirname, 'templates/description.art'), {
+                description: item.des,
+                enclosure_url,
+                enclosure_type,
+            }),
+            author: data.anchorpersons,
+            pubDate: parseDate(item.publishTime),
+            enclosure_url,
+            enclosure_type,
+            enclosure_length: item.fileSize,
+            itunes_duration: item.duration,
+            itunes_item_image: iconUrl,
+        };
+    });
+
+    ctx.state.data = {
+        title: `云听 - ${data.columnName}`,
+        link: currentUrl,
+        item: items,
+        image: iconUrl,
+        itunes_author: data.anchorpersons,
+        description: data.descriptions ?? data.descriptionSimple,
+        itunes_category: data.atypeInfo.map((c) => c.categoryName).join(),
+    };
+};

package/lib/v2/radio/index.js

@@ -5,12 +5,12 @@ const path = require('path');
 
 module.exports = async (ctx) => {
     const id = ctx.params.id;
+    const size = ctx.query.limit ?? '100';
 
-    const rootUrl = 'http://www.radio.cn';
-    const currentUrl = `${rootUrl}/pc-portal/sanji/detail.html?columnId=${id}`;
-
+    const rootUrl = 'https://www.radio.cn';
     const apiRootUrl = 'http://tacc.radio.cn';
-    const apiUrl = `${apiRootUrl}/pcpages/odchannelpages?od_id=${id}&start=1&rows=50`;
+    const apiUrl = `${apiRootUrl}/pcpages/odchannelpages?od_id=${id}&start=1&rows=${size}`;
+    const currentUrl = `${rootUrl}/pc-portal/sanji/detail.html?columnId=${id}`;
 
     const response = await got({
         method: 'get',
@@ -18,24 +18,31 @@ module.exports = async (ctx) => {
     });
 
     if (/^\(.*\)$/.test(response.data)) {
-        response.data = JSON.parse(response.data)[0];
+        response.data = JSON.parse(response.data.match(/^\((.*)\)$/)[1]);
     }
 
     const data = response.data.data;
 
-    const items = data.program.map((item) => ({
-        guid: item.id,
-        title: item.name,
-        link: item.streams[0].url,
-        description: `${item.description}${art(path.join(__dirname, 'templates/description.art'), {
-            streams: item.streams,
-        })}`,
-        pubDate: parseDate(item.onlinetime),
-        enclosure_url: item.streams[0].url,
-        enclosure_type: 'audio/x-m4a',
-        itunes_duration: item.duration,
-        itunes_item_image: data.odchannel.imageUrl[0].url,
-    }));
+    const items = data.program.map((item) => {
+        const enclosure_url = item.streams[0].url;
+        const enclosure_type = `audio/${enclosure_url.match(/\.(\w+)$/)[1]}`;
+
+        return {
+            guid: item.id,
+            title: item.name,
+            link: item.streams[0].url,
+            description: art(path.join(__dirname, 'templates/description.art'), {
+                description: item.description,
+                enclosure_url,
+                enclosure_type,
+            }),
+            pubDate: parseDate(item.onlinetime),
+            enclosure_url,
+            enclosure_type,
+            itunes_duration: item.duration,
+            itunes_item_image: data.odchannel.imageUrl[0].url,
+        };
+    });
 
     ctx.state.data = {
         title: `云听 - ${data.odchannel.name}`,

package/lib/v2/radio/maintainer.js

@@ -1,3 +1,5 @@
 module.exports = {
+    '/album/:id': ['nczitzk'],
+    '/zhibo/:id': ['nczitzk'],
     '/:id': ['kt286', 'nczitzk'],
 };

package/lib/v2/radio/radar.js

@@ -3,11 +3,23 @@ module.exports = {
         _name: '云听',
         '.': [
             {
-                title: '电台节目',
-                docs: 'https://docs.rsshub.app/multimedia.html#yun-ting-dian-tai-jie-mu',
-                source: ['/pc-portal/sanji/detail.html', '/'],
+                title: '专辑',
+                docs: 'https://docs.rsshub.app/multimedia.html#yun-ting-zhuan-ji',
+                source: ['/pc-portal/sanji/detail.html', '/share/albumDetail', '/'],
+                target: (params, url) => `/radio/album/${new URL(url).searchParams.get('columnId')}`,
+            },
+            {
+                title: '节目',
+                docs: 'https://docs.rsshub.app/multimedia.html#yun-ting-jie-mu',
+                source: ['/pc-portal/sanji/detail.html', '/share/albumDetail', '/'],
                 target: (params, url) => `/radio/${new URL(url).searchParams.get('columnId')}`,
             },
+            {
+                title: '直播',
+                docs: 'https://docs.rsshub.app/multimedia.html#yun-ting-zhi-bo',
+                source: ['/pc-portal/sanji/zhibo_2.html', '/'],
+                target: (params, url) => `/radio/zhibo/${new URL(url).searchParams.get('name')}`,
+            },
         ],
     },
 };

package/lib/v2/radio/router.js

@@ -1,3 +1,5 @@
 module.exports = function (router) {
+    router.get('/album/:id', require('./album'));
+    router.get('/zhibo/:id', require('./zhibo'));
     router.get('/:id', require('./index'));
 };

package/lib/v2/radio/templates/description.art

@@ -1,5 +1,8 @@
+{{ if description }}
+<p>{{ description }}</p>
+{{ /if }}
+{{ if enclosure_url && enclosure_type }}
 <audio controls="controls">
-{{each streams stream}}
-<source src="{{ stream.url }}" type="audio/x-m4a">
-{{/each}}
-</audio>
+<source src="{{ enclosure_url }}" type="{{ enclosure_type }}">
+</audio>
+{{ /if }}

package/lib/v2/radio/zhibo.js

@@ -0,0 +1,73 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+const CryptoJS = require('crypto-js');
+
+module.exports = async (ctx) => {
+    const KEY = 'f0fc4c668392f9f9a447e48584c214ee';
+
+    const id = ctx.params.id;
+    const size = ctx.query.limit ?? '100';
+
+    const params = `columnId=${id}&pageNo=0&pageSize=${size}`;
+
+    const rootUrl = 'https://www.radio.cn';
+    const apiRootUrl = 'https://ytmsout.radio.cn';
+
+    const iconUrl = `${rootUrl}/pc-portal/image/icon_32.jpg`;
+    const currentUrl = `${rootUrl}/pc-portal/sanji/zhibo_2.html?name=${id}`;
+    const apiUrl = `${apiRootUrl}/web/appProgram/pageByColumn?${params}`;
+
+    const timestamp = new Date().getTime();
+    const sign = CryptoJS.MD5(`${params}&timestamp=${timestamp}&key=${KEY}`).toString().toUpperCase();
+
+    const response = await got({
+        method: 'get',
+        url: apiUrl,
+        headers: {
+            sign,
+            timestamp,
+            'Content-Type': 'application/json',
+            equipmentId: '0000',
+            platformCode: 'WEB',
+        },
+    });
+
+    const data = response.data.data.data;
+
+    const items = data.map((item) => {
+        let enclosure_url = item.playUrlHigh ?? item.playUrlLow;
+        enclosure_url = /\.m3u8$/.test(enclosure_url) ? item.downloadUrl : enclosure_url;
+
+        const enclosure_type = `audio/${enclosure_url.match(/\.(\w+)$/)[1]}`;
+
+        const date = new Date(item.startTime);
+        const dateString = `${date.getFullYear()}-${date.getMonth() + 1}-${date.getDate()}`;
+
+        return {
+            guid: item.id,
+            title: `${dateString} ${item.name}`,
+            link: enclosure_url,
+            description: art(path.join(__dirname, 'templates/description.art'), {
+                description: item.des,
+                enclosure_url,
+                enclosure_type,
+            }),
+            pubDate: parseDate(item.startTime),
+            enclosure_url,
+            enclosure_type,
+            itunes_duration: item.durationStr,
+            itunes_item_image: iconUrl,
+        };
+    });
+
+    ctx.state.data = {
+        title: `云听 - ${data[0].name}`,
+        link: currentUrl,
+        item: items,
+        image: iconUrl,
+        itunes_author: 'radio.cn',
+        description: data[0].des,
+    };
+};

package/lib/v2/researchgate/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/publications/:username': ['nczitzk'],
+};

package/lib/v2/researchgate/publications.js

@@ -0,0 +1,70 @@
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id;
+
+    const rootUrl = 'https://www.researchgate.net';
+    const currentUrl = `${rootUrl}/profile/${id}`;
+    const browser = await require('@/utils/puppeteer')();
+    const page = await browser.newPage();
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
+    await page.goto(currentUrl);
+    const response = await page.evaluate(() => document.documentElement.innerHTML);
+    await page.close();
+
+    const $ = cheerio.load(response);
+
+    const list = $('div[itemprop="headline"] a')
+        .toArray()
+        .slice(0, ctx.query.limit ? Number(ctx.query.limit) : 15)
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.text(),
+                link: item.attr('href'),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const page = await browser.newPage();
+                await page.setRequestInterception(true);
+                page.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                });
+                await page.goto(item.link);
+                const detailResponse = await page.evaluate(() => document.documentElement.innerHTML);
+                await page.close();
+                const content = cheerio.load(detailResponse);
+
+                item.doi = content('meta[property="citation_doi"]').attr('content');
+                item.pubDate = parseDate(content('meta[property="citation_publication_date"]').attr('content'));
+
+                const authors = [];
+
+                content('meta[property="citation_author"]').each(function () {
+                    authors.push(content(this).attr('content'));
+                });
+
+                item.author = authors.join(', ');
+
+                item.description = content('div[itemprop="description"]').html();
+
+                return item;
+            })
+        )
+    );
+
+    await browser.close();
+
+    ctx.state.data = {
+        title: `${$('meta[property="profile:username"]').attr('content')}'s Publications - ResearchGate`,
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/researchgate/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'researchgate.net': {
+        _name: 'ResearchGate',
+        '.': [
+            {
+                title: 'Publications',
+                docs: 'https://docs.rsshub.app/study.html#researchgate',
+                source: ['/profile/:username'],
+                target: '/researchgate/publications/:username',
+            },
+        ],
+    },
+};

package/lib/v2/researchgate/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/publications/:id', require('./publications'));
+};

package/lib/v2/reuters/common.js

@@ -1,6 +1,8 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
 
 module.exports = async (ctx) => {
     const category = ctx.params.category;
@@ -12,15 +14,15 @@ module.exports = async (ctx) => {
     const $ = cheerio.load(response.data);
 
     let list = $('.media-story-card__body__3tRWy a.media-story-card__heading__eqhp9')
-        .map((_, item) => {
+        .toArray()
+        .map((item) => {
             item = $(item);
             item.find('span.visually-hidden__hidden__2qXMW').remove();
             return {
                 title: item.text(),
                 link: rootUrl + item.prop('href'),
             };
-        })
-        .get();
+        });
     if (!list.length) {
         const metadata = $('script#fusion-metadata').html();
         const metadataObj = JSON.parse(metadata.match(/Fusion.globalContent=(\{[\s\S]*?});/)[1]);
@@ -36,10 +38,19 @@ module.exports = async (ctx) => {
                 const detailResponse = await got(item.link);
                 const content = cheerio.load(detailResponse.data);
 
-                item.title = content('meta[property="og:title"]').attr('content') || item.title;
-                item.description = content('p[data-testid="paragraph-0"]').text();
-                item.pubDate = parseDate(content('meta[name="article:published_time"]').attr('content'));
-                item.author = content('meta[name="article:author"]').attr('content');
+                const data = JSON.parse(
+                    content('script#fusion-metadata')
+                        .text()
+                        .match(/Fusion.globalContent=(\{[\s\S]*?});/)[1]
+                );
+
+                item.title = data.result.title || item.title;
+                item.description = art(path.join(__dirname, 'templates/description.art'), {
+                    result: data.result,
+                });
+                item.pubDate = parseDate(data.result.display_time);
+                item.author = data.result.authors.map((author) => author.name).join(', ');
+                item.category = data.result.taxonomy.keywords;
 
                 return item;
             })
@@ -47,7 +58,9 @@ module.exports = async (ctx) => {
     );
 
     ctx.state.data = {
-        title: topic ? `Reuters - ${category} - ${topic}` : `Reuters - ${category}`,
+        title: $('head title').text(),
+        description: $('head meta[name=description]').attr('content'),
+        image: 'https://www.reuters.com/pf/resources/images/reuters/logo-vertical-default-512x512.png?d=116',
         link: currentUrl,
         item: items,
     };

package/lib/v2/reuters/investigates.js

@@ -22,7 +22,7 @@ module.exports = async (ctx) => {
                 const content = cheerio.load(detailResponse.data);
 
                 item.title = content('title').text();
-                item.description = content('#paragraph-0').text();
+                item.description = content('article.special-report').html();
                 item.pubDate = parseDate(content('time[itemprop="datePublished"]').attr('datetime'));
                 item.author = content('meta[property="og:article:publisher"]').attr('content');
 

package/lib/v2/reuters/templates/description.art

@@ -0,0 +1,44 @@
+{{ if result.summary }}
+Summary:
+<ul>{{ each result.summary s }}
+<li>{{ s.description }}</li>
+{{ /each }}</ul>
+{{ /if }}
+
+{{ if result.related_content }}
+{{ each result.related_content.galleries g }}
+    {{ each g.content_elements e }}
+        {{ if e.type === 'image' }}
+            <figure><img src="{{ e.url }}" alt="{{ e.alt_text }}">
+            <figcaption>{{ e.caption }}</figcaption>
+            </figure>
+        {{ /if }}
+    {{ /each }}
+{{ /each }}
+{{ each result.related_content.images i }}
+    {{ if i.type === 'image' }}
+        <figure><img src="{{ i.url }}" alt="{{ i.alt_text }}">
+        <figcaption>{{ i.caption }}</figcaption>
+        </figure>
+    {{ /if }}
+{{ /each }}
+{{ each result.related_content.galleries v }}
+    {{ each v.content_elements c }}
+        {{ if c.type === 'video' }}
+            <video controls poster="{{ c.thumbnail.url }}">
+            <source src="{{ c.source.mp4 }}" type="video/mp4" />
+            </video>
+            {{ c.description }}
+        {{ /if }}
+    {{ /each }}
+{{ /each }}
+{{ /if }}
+
+{{ if result.content_elements }}
+{{ each result.content_elements e }}
+    {{ if e.type === 'paragraph' }}<p>{{@ e.content }}</p>{{ /if }}
+    {{ if e.type === 'header' }}<h{{ e.level }}>{{@ e.content }}</h{{ e.level }}>{{ /if }}
+{{ /each }}
+{{ /if }}
+
+{{ if result.sign_off }}<span>{{ result.sign_off }}</span>{{ /if }}

package/lib/v2/saraba1st/digest.js

@@ -0,0 +1,79 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const config = require('@/config').value;
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const tid = ctx.params.tid;
+    const cookieString = config.saraba1st.cookie ? config.saraba1st.cookie : '';
+    const res = await got('https://bbs.saraba1st.com/2b/' + tid + '.html', {
+        headers: {
+            Cookie: cookieString,
+        },
+    });
+    const $ = cheerio.load(res.data);
+    const title = $('head title').text().replace(/-.*/, '');
+    const list = $('#threadlisttableid tbody[id^="normalthread_"] tr');
+    const count = list
+        .slice(0, ctx.query.limit ? Number(ctx.query.limit) : 20)
+        .toArray()
+        .map((each) => {
+            each = $(each);
+            const floor = each.find('th.new a.s.xst').text();
+            const floorUrl = each.find('th.new a.s.xst').attr('href');
+            return {
+                title: `${title}:${floor}`,
+                link: new URL(floorUrl, 'https://bbs.saraba1st.com/2b/').href,
+                author: each.find('td.by cite').text(),
+                pubDate: timezone(parseDate(each.find('td.by em').first().text()), +8),
+            };
+        });
+
+    const resultItems = await Promise.all(
+        count.map((i) =>
+            ctx.cache.tryGet(i.link, async () => {
+                i.description = await fetchContent(i.link);
+                return i;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `Stage1 论坛 - ${title}`,
+        link: `https://bbs.saraba1st.com/2b/${tid}.html`,
+        //        item: await resultItems,
+        item: resultItems,
+    };
+};
+
+async function fetchContent(url) {
+    // Fetch the subpageinof
+    const cookieString = config.saraba1st.cookie ? config.saraba1st.cookie : '';
+    const subres = await got(url, {
+        headers: {
+            Cookie: cookieString,
+        },
+    });
+    const subind = cheerio.load(subres.data);
+    const stubS = subind('<div>');
+    subind('#postlist')
+        .find('div[id*="post_"] ')
+        .each(function () {
+            if (subind(this).find('td[id*="postmessage_"]').length > 0) {
+                const section = art(path.join(__dirname, 'templates/digest.art'), {
+                    author: {
+                        link: subind(this).find('.pls.favatar div.authi a').attr('href'),
+                        name: subind(this).find('.pls.favatar div.authi').text(),
+                        postinfo: subind(this).find('div.authi em[id*=authorposton]').text(),
+                    },
+                    msg: subind(this).find('td[id*="postmessage_"]').html(),
+                });
+                stubS.append(section);
+            }
+        });
+
+    return stubS.html();
+}

package/lib/v2/saraba1st/maintainer.js

@@ -1,3 +1,4 @@
 module.exports = {
+    '/digest/:tid': ['shinemoon'],
     '/thread/:tid': ['zengxs'],
 };

package/lib/v2/saraba1st/radar.js

@@ -3,12 +3,21 @@ module.exports = {
         _name: 'Saraba1st',
         bbs: [
             {
-                title: '帖子',
+                title: '论坛',
                 docs: 'https://docs.rsshub.app/bbs.html#saraba1st',
                 source: '/2b/:id',
                 target: (params) => {
-                    const id = params.id.includes('thread') ? params.id.split('-')[1] : '';
-                    return id ? `/saraba1st/thread/${id}` : '';
+                    let id = params.id;
+                    // For Digest
+                    if (id.match('forum') !== null) {
+                        id = id.substring(0, id.length - 5);
+                        return `/saraba1st/digest/${id}`;
+                    }
+                    // For Thread
+                    else if (id.match('thread') !== null) {
+                        id = params.id.includes('thread') ? params.id.split('-')[1] : '';
+                        return id ? `/saraba1st/thread/${id}` : '';
+                    }
                 },
             },
         ],

package/lib/v2/saraba1st/router.js

@@ -1,3 +1,4 @@
 module.exports = (router) => {
     router.get('/thread/:tid', require('./thread'));
+    router.get('/digest/:tid', require('./digest'));
 };

package/lib/v2/saraba1st/templates/digest.art

@@ -0,0 +1,5 @@
+<div class="quoted" style="background:rgba(220,220,220,0.3);">
+    <a style="text-decoration: none;" href="https://bbs.saraba1st.com/2b/{{ author.link }}">{{ author.name }}</a>
+    <span style="text-decoration: none;color:#aaa;">{{ author.postinfo }}</span>
+</div>
+<div class="content" style="margin-bottom:20px!important;">{{@ msg }}</div>

package/lib/v2/science/current.js

@@ -36,6 +36,7 @@ module.exports = async (ctx) => {
         description: `Current Issue of ${pageTitleName}`,
         image: `${baseUrl}/apple-touch-icon.png`,
         link: pageURL,
+        language: 'en-US',
         item: items,
     };
 };

package/lib/v2/science/maintainer.js

@@ -1,5 +1,5 @@
 module.exports = {
-    '/cover': ['yech1990', 'TonyRL'],
-    '/current/:journal?': ['yech1990', 'TonyRL'],
-    '/early/:journal?': ['yech1990', 'TonyRL'],
+    '/cover': ['y9c', 'TonyRL'],
+    '/current/:journal?': ['y9c', 'TonyRL'],
+    '/early/:journal?': ['y9c', 'TonyRL'],
 };