rsshub 1.0.0-master.fe50f9c → 1.0.0-master.fe560d2

package/lib/v2/radio/album.js

@@ -0,0 +1,106 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+const CryptoJS = require('crypto-js');
+
+module.exports = async (ctx) => {
+    const KEY = 'f0fc4c668392f9f9a447e48584c214ee';
+
+    const id = ctx.params.id;
+    const size = ctx.query.limit ? parseInt(ctx.query.limit) : 100;
+
+    const rootUrl = 'https://www.radio.cn';
+    const apiRootUrl = 'https://ytmsout.radio.cn';
+    const detailRootUrl = 'https://ytapi.radio.cn';
+
+    const iconUrl = `${rootUrl}/pc-portal/image/icon_32.jpg`;
+    const currentUrl = `${rootUrl}/pc-portal/sanji/detail.html?columnId=${id}`;
+    const detailUrl = `${detailRootUrl}/ytsrv/srv/wifimusicbox/demand/detail`;
+
+    const detailResponse = await got({
+        method: 'post',
+        url: detailUrl,
+        form: {
+            pageIndex: '0',
+            sortType: '2',
+            mobileId: '',
+            providerCode: '25010',
+            pid: id,
+            paySongFlag: '1',
+            richText: '1',
+            h5flag: '1',
+        },
+        headers: {
+            'Content-Type': 'application/x-www-form-urlencoded',
+            version: '4.0.0',
+            providerCode: '25010',
+            equipmentSource: 'WEB',
+        },
+    });
+
+    const data = detailResponse.data;
+
+    const count = data.count;
+
+    let pageNo = 1,
+        pageSize = count - size;
+
+    if (pageSize <= 0) {
+        pageNo = 0;
+        pageSize = count;
+    }
+
+    const params = `albumId=${id}&pageNo=${pageNo}&pageSize=${pageSize}`;
+    const apiUrl = `${apiRootUrl}/web/appSingle/pageByAlbum?${params}`;
+
+    const timestamp = new Date().getTime();
+    const sign = CryptoJS.MD5(`${params}&timestamp=${timestamp}&key=${KEY}`).toString().toUpperCase();
+
+    const response = await got({
+        method: 'get',
+        url: apiUrl,
+        headers: {
+            sign,
+            timestamp,
+            'Content-Type': 'application/json',
+            equipmentId: '0000',
+            platformCode: 'WEB',
+        },
+    });
+
+    const items = response.data.data.data.map((item) => {
+        let enclosure_url = item.playUrlHigh ?? item.playUrlLow;
+        enclosure_url = /\.m3u8$/.test(enclosure_url) ? item.downloadUrl : enclosure_url;
+
+        const enclosure_type = `audio/${enclosure_url.match(/\.(\w+)$/)[1]}`;
+
+        return {
+            guid: item.id,
+            title: item.name,
+            link: enclosure_url,
+            description: art(path.join(__dirname, 'templates/description.art'), {
+                description: item.des,
+                enclosure_url,
+                enclosure_type,
+            }),
+            author: data.anchorpersons,
+            pubDate: parseDate(item.publishTime),
+            enclosure_url,
+            enclosure_type,
+            enclosure_length: item.fileSize,
+            itunes_duration: item.duration,
+            itunes_item_image: iconUrl,
+        };
+    });
+
+    ctx.state.data = {
+        title: `云听 - ${data.columnName}`,
+        link: currentUrl,
+        item: items,
+        image: iconUrl,
+        itunes_author: data.anchorpersons,
+        description: data.descriptions ?? data.descriptionSimple,
+        itunes_category: data.atypeInfo.map((c) => c.categoryName).join(),
+    };
+};

package/lib/v2/radio/index.js

@@ -5,12 +5,12 @@ const path = require('path');
 
 module.exports = async (ctx) => {
     const id = ctx.params.id;
+    const size = ctx.query.limit ?? '100';
 
-    const rootUrl = 'http://www.radio.cn';
-    const currentUrl = `${rootUrl}/pc-portal/sanji/detail.html?columnId=${id}`;
-
+    const rootUrl = 'https://www.radio.cn';
     const apiRootUrl = 'http://tacc.radio.cn';
-    const apiUrl = `${apiRootUrl}/pcpages/odchannelpages?od_id=${id}&start=1&rows=50`;
+    const apiUrl = `${apiRootUrl}/pcpages/odchannelpages?od_id=${id}&start=1&rows=${size}`;
+    const currentUrl = `${rootUrl}/pc-portal/sanji/detail.html?columnId=${id}`;
 
     const response = await got({
         method: 'get',
@@ -18,24 +18,31 @@ module.exports = async (ctx) => {
     });
 
     if (/^\(.*\)$/.test(response.data)) {
-        response.data = JSON.parse(response.data)[0];
+        response.data = JSON.parse(response.data.match(/^\((.*)\)$/)[1]);
     }
 
     const data = response.data.data;
 
-    const items = data.program.map((item) => ({
-        guid: item.id,
-        title: item.name,
-        link: item.streams[0].url,
-        description: `${item.description}${art(path.join(__dirname, 'templates/description.art'), {
-            streams: item.streams,
-        })}`,
-        pubDate: parseDate(item.onlinetime),
-        enclosure_url: item.streams[0].url,
-        enclosure_type: 'audio/x-m4a',
-        itunes_duration: item.duration,
-        itunes_item_image: data.odchannel.imageUrl[0].url,
-    }));
+    const items = data.program.map((item) => {
+        const enclosure_url = item.streams[0].url;
+        const enclosure_type = `audio/${enclosure_url.match(/\.(\w+)$/)[1]}`;
+
+        return {
+            guid: item.id,
+            title: item.name,
+            link: item.streams[0].url,
+            description: art(path.join(__dirname, 'templates/description.art'), {
+                description: item.description,
+                enclosure_url,
+                enclosure_type,
+            }),
+            pubDate: parseDate(item.onlinetime),
+            enclosure_url,
+            enclosure_type,
+            itunes_duration: item.duration,
+            itunes_item_image: data.odchannel.imageUrl[0].url,
+        };
+    });
 
     ctx.state.data = {
         title: `云听 - ${data.odchannel.name}`,

package/lib/v2/radio/maintainer.js

@@ -1,3 +1,5 @@
 module.exports = {
+    '/album/:id': ['nczitzk'],
+    '/zhibo/:id': ['nczitzk'],
     '/:id': ['kt286', 'nczitzk'],
 };

package/lib/v2/radio/radar.js

@@ -3,11 +3,23 @@ module.exports = {
         _name: '云听',
         '.': [
             {
-                title: '电台节目',
-                docs: 'https://docs.rsshub.app/multimedia.html#yun-ting-dian-tai-jie-mu',
-                source: ['/pc-portal/sanji/detail.html', '/'],
+                title: '专辑',
+                docs: 'https://docs.rsshub.app/multimedia.html#yun-ting-zhuan-ji',
+                source: ['/pc-portal/sanji/detail.html', '/share/albumDetail', '/'],
+                target: (params, url) => `/radio/album/${new URL(url).searchParams.get('columnId')}`,
+            },
+            {
+                title: '节目',
+                docs: 'https://docs.rsshub.app/multimedia.html#yun-ting-jie-mu',
+                source: ['/pc-portal/sanji/detail.html', '/share/albumDetail', '/'],
                 target: (params, url) => `/radio/${new URL(url).searchParams.get('columnId')}`,
             },
+            {
+                title: '直播',
+                docs: 'https://docs.rsshub.app/multimedia.html#yun-ting-zhi-bo',
+                source: ['/pc-portal/sanji/zhibo_2.html', '/'],
+                target: (params, url) => `/radio/zhibo/${new URL(url).searchParams.get('name')}`,
+            },
         ],
     },
 };

package/lib/v2/radio/router.js

@@ -1,3 +1,5 @@
 module.exports = function (router) {
+    router.get('/album/:id', require('./album'));
+    router.get('/zhibo/:id', require('./zhibo'));
     router.get('/:id', require('./index'));
 };

package/lib/v2/radio/templates/description.art

@@ -1,5 +1,8 @@
+{{ if description }}
+<p>{{ description }}</p>
+{{ /if }}
+{{ if enclosure_url && enclosure_type }}
 <audio controls="controls">
-{{each streams stream}}
-<source src="{{ stream.url }}" type="audio/x-m4a">
-{{/each}}
-</audio>
+<source src="{{ enclosure_url }}" type="{{ enclosure_type }}">
+</audio>
+{{ /if }}

package/lib/v2/radio/zhibo.js

@@ -0,0 +1,73 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+const CryptoJS = require('crypto-js');
+
+module.exports = async (ctx) => {
+    const KEY = 'f0fc4c668392f9f9a447e48584c214ee';
+
+    const id = ctx.params.id;
+    const size = ctx.query.limit ?? '100';
+
+    const params = `columnId=${id}&pageNo=0&pageSize=${size}`;
+
+    const rootUrl = 'https://www.radio.cn';
+    const apiRootUrl = 'https://ytmsout.radio.cn';
+
+    const iconUrl = `${rootUrl}/pc-portal/image/icon_32.jpg`;
+    const currentUrl = `${rootUrl}/pc-portal/sanji/zhibo_2.html?name=${id}`;
+    const apiUrl = `${apiRootUrl}/web/appProgram/pageByColumn?${params}`;
+
+    const timestamp = new Date().getTime();
+    const sign = CryptoJS.MD5(`${params}&timestamp=${timestamp}&key=${KEY}`).toString().toUpperCase();
+
+    const response = await got({
+        method: 'get',
+        url: apiUrl,
+        headers: {
+            sign,
+            timestamp,
+            'Content-Type': 'application/json',
+            equipmentId: '0000',
+            platformCode: 'WEB',
+        },
+    });
+
+    const data = response.data.data.data;
+
+    const items = data.map((item) => {
+        let enclosure_url = item.playUrlHigh ?? item.playUrlLow;
+        enclosure_url = /\.m3u8$/.test(enclosure_url) ? item.downloadUrl : enclosure_url;
+
+        const enclosure_type = `audio/${enclosure_url.match(/\.(\w+)$/)[1]}`;
+
+        const date = new Date(item.startTime);
+        const dateString = `${date.getFullYear()}-${date.getMonth() + 1}-${date.getDate()}`;
+
+        return {
+            guid: item.id,
+            title: `${dateString} ${item.name}`,
+            link: enclosure_url,
+            description: art(path.join(__dirname, 'templates/description.art'), {
+                description: item.des,
+                enclosure_url,
+                enclosure_type,
+            }),
+            pubDate: parseDate(item.startTime),
+            enclosure_url,
+            enclosure_type,
+            itunes_duration: item.durationStr,
+            itunes_item_image: iconUrl,
+        };
+    });
+
+    ctx.state.data = {
+        title: `云听 - ${data[0].name}`,
+        link: currentUrl,
+        item: items,
+        image: iconUrl,
+        itunes_author: 'radio.cn',
+        description: data[0].des,
+    };
+};

package/lib/v2/reactnewsletter/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '': ['meixger'],
+    '/': ['meixger'],
+};

package/lib/v2/reactnewsletter/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'bytes.dev': {
+        _name: 'ui.dev',
+        '.': [
+            {
+                title: 'React Newsletter',
+                docs: 'https://docs.rsshub.app/en/programming.html#ui-dev-bytes',
+                source: ['/issues', '/'],
+                target: '/reactnewsletter',
+            },
+        ],
+    },
+};

package/lib/v2/reactnewsletter/reactnewsletter.js

@@ -0,0 +1,25 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const currentURL = 'https://reactnewsletter.com/issues';
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const resp = await got(currentURL);
+    const $ = cheerio.load(resp.data);
+    const text = $('script#__NEXT_DATA__').text();
+    const json = JSON.parse(text);
+
+    const items = json.props.pageProps.issues.map((item) => ({
+        title: item.title,
+        pubDate: parseDate(item.date),
+        description: item.summary,
+        link: `/issues/${item.slug}`,
+    }));
+
+    ctx.state.data = {
+        title: 'reactnewsletter.dev',
+        description: 'Stay up to date on the latest React news, tutorials, resources, and more. Delivered every Tuesday, for free.',
+        link: currentURL,
+        item: items,
+    };
+};

package/lib/v2/reactnewsletter/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/', require('./reactnewsletter'));
+};

package/lib/v2/researchgate/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/publications/:username': ['nczitzk'],
+};

package/lib/v2/researchgate/publications.js

@@ -0,0 +1,70 @@
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id;
+
+    const rootUrl = 'https://www.researchgate.net';
+    const currentUrl = `${rootUrl}/profile/${id}`;
+    const browser = await require('@/utils/puppeteer')();
+    const page = await browser.newPage();
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
+    await page.goto(currentUrl);
+    const response = await page.evaluate(() => document.documentElement.innerHTML);
+    await page.close();
+
+    const $ = cheerio.load(response);
+
+    const list = $('div[itemprop="headline"] a')
+        .toArray()
+        .slice(0, ctx.query.limit ? Number(ctx.query.limit) : 15)
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.text(),
+                link: item.attr('href'),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const page = await browser.newPage();
+                await page.setRequestInterception(true);
+                page.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                });
+                await page.goto(item.link);
+                const detailResponse = await page.evaluate(() => document.documentElement.innerHTML);
+                await page.close();
+                const content = cheerio.load(detailResponse);
+
+                item.doi = content('meta[property="citation_doi"]').attr('content');
+                item.pubDate = parseDate(content('meta[property="citation_publication_date"]').attr('content'));
+
+                const authors = [];
+
+                content('meta[property="citation_author"]').each(function () {
+                    authors.push(content(this).attr('content'));
+                });
+
+                item.author = authors.join(', ');
+
+                item.description = content('div[itemprop="description"]').html();
+
+                return item;
+            })
+        )
+    );
+
+    await browser.close();
+
+    ctx.state.data = {
+        title: `${$('meta[property="profile:username"]').attr('content')}'s Publications - ResearchGate`,
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/researchgate/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'researchgate.net': {
+        _name: 'ResearchGate',
+        '.': [
+            {
+                title: 'Publications',
+                docs: 'https://docs.rsshub.app/study.html#researchgate',
+                source: ['/profile/:username'],
+                target: '/researchgate/publications/:username',
+            },
+        ],
+    },
+};

package/lib/v2/researchgate/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/publications/:id', require('./publications'));
+};

package/lib/v2/reuters/common.js

@@ -1,6 +1,8 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
 
 module.exports = async (ctx) => {
     const category = ctx.params.category;
@@ -12,15 +14,15 @@ module.exports = async (ctx) => {
     const $ = cheerio.load(response.data);
 
     let list = $('.media-story-card__body__3tRWy a.media-story-card__heading__eqhp9')
-        .map((_, item) => {
+        .toArray()
+        .map((item) => {
             item = $(item);
             item.find('span.visually-hidden__hidden__2qXMW').remove();
             return {
                 title: item.text(),
                 link: rootUrl + item.prop('href'),
             };
-        })
-        .get();
+        });
     if (!list.length) {
         const metadata = $('script#fusion-metadata').html();
         const metadataObj = JSON.parse(metadata.match(/Fusion.globalContent=(\{[\s\S]*?});/)[1]);
@@ -36,10 +38,19 @@ module.exports = async (ctx) => {
                 const detailResponse = await got(item.link);
                 const content = cheerio.load(detailResponse.data);
 
-                item.title = content('meta[property="og:title"]').attr('content') || item.title;
-                item.description = content('p[data-testid="paragraph-0"]').text();
-                item.pubDate = parseDate(content('meta[name="article:published_time"]').attr('content'));
-                item.author = content('meta[name="article:author"]').attr('content');
+                const data = JSON.parse(
+                    content('script#fusion-metadata')
+                        .text()
+                        .match(/Fusion.globalContent=(\{[\s\S]*?});/)[1]
+                );
+
+                item.title = data.result.title || item.title;
+                item.description = art(path.join(__dirname, 'templates/description.art'), {
+                    result: data.result,
+                });
+                item.pubDate = parseDate(data.result.display_time);
+                item.author = data.result.authors.map((author) => author.name).join(', ');
+                item.category = data.result.taxonomy.keywords;
 
                 return item;
             })
@@ -47,7 +58,9 @@ module.exports = async (ctx) => {
     );
 
     ctx.state.data = {
-        title: topic ? `Reuters - ${category} - ${topic}` : `Reuters - ${category}`,
+        title: $('head title').text(),
+        description: $('head meta[name=description]').attr('content'),
+        image: 'https://www.reuters.com/pf/resources/images/reuters/logo-vertical-default-512x512.png?d=116',
         link: currentUrl,
         item: items,
     };

package/lib/v2/reuters/investigates.js

@@ -22,7 +22,7 @@ module.exports = async (ctx) => {
                 const content = cheerio.load(detailResponse.data);
 
                 item.title = content('title').text();
-                item.description = content('#paragraph-0').text();
+                item.description = content('article.special-report').html();
                 item.pubDate = parseDate(content('time[itemprop="datePublished"]').attr('datetime'));
                 item.author = content('meta[property="og:article:publisher"]').attr('content');
 

package/lib/v2/reuters/templates/description.art

@@ -0,0 +1,44 @@
+{{ if result.summary }}
+Summary:
+<ul>{{ each result.summary s }}
+<li>{{ s.description }}</li>
+{{ /each }}</ul>
+{{ /if }}
+
+{{ if result.related_content }}
+{{ each result.related_content.galleries g }}
+    {{ each g.content_elements e }}
+        {{ if e.type === 'image' }}
+            <figure><img src="{{ e.url }}" alt="{{ e.alt_text }}">
+            <figcaption>{{ e.caption }}</figcaption>
+            </figure>
+        {{ /if }}
+    {{ /each }}
+{{ /each }}
+{{ each result.related_content.images i }}
+    {{ if i.type === 'image' }}
+        <figure><img src="{{ i.url }}" alt="{{ i.alt_text }}">
+        <figcaption>{{ i.caption }}</figcaption>
+        </figure>
+    {{ /if }}
+{{ /each }}
+{{ each result.related_content.galleries v }}
+    {{ each v.content_elements c }}
+        {{ if c.type === 'video' }}
+            <video controls poster="{{ c.thumbnail.url }}">
+            <source src="{{ c.source.mp4 }}" type="video/mp4" />
+            </video>
+            {{ c.description }}
+        {{ /if }}
+    {{ /each }}
+{{ /each }}
+{{ /if }}
+
+{{ if result.content_elements }}
+{{ each result.content_elements e }}
+    {{ if e.type === 'paragraph' }}<p>{{@ e.content }}</p>{{ /if }}
+    {{ if e.type === 'header' }}<h{{ e.level }}>{{@ e.content }}</h{{ e.level }}>{{ /if }}
+{{ /each }}
+{{ /if }}
+
+{{ if result.sign_off }}<span>{{ result.sign_off }}</span>{{ /if }}

package/lib/v2/ruancan/index.js

@@ -16,19 +16,19 @@ module.exports = async (ctx) => {
 
     const $ = cheerio.load(response.data);
 
-    const list = $('.item-title a')
-        .map((_, item) => {
+    let items = $('.entry-title a')
+        .toArray()
+        .map((item) => {
             item = $(item);
 
             return {
                 title: item.text(),
                 link: item.attr('href'),
             };
-        })
-        .get();
+        });
 
-    const items = await Promise.all(
-        list.map((item) =>
+    items = await Promise.all(
+        items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 const detailResponse = await got({
                     method: 'get',
@@ -37,9 +37,14 @@ module.exports = async (ctx) => {
 
                 const content = cheerio.load(detailResponse.data);
 
-                item.author = content('.author-name').text();
-                item.pubDate = parseDate(content('.published').attr('datetime'));
+                content('.entry-content .heateorSssClear').first().nextAll().remove();
+                content('.adsbygoogle, .heateorSssClear').remove();
+
                 item.description = content('.entry-content').html();
+                item.category = content('.cat-links a')
+                    .toArray()
+                    .map((a) => content(a).text());
+                item.pubDate = parseDate(content('meta[property="article:published_time"]').attr('content'));
 
                 return item;
             })