rsshub 1.0.0-master.f7cdf8b → 1.0.0-master.f812e91

package/lib/v2/nature/utils.js

@@ -1,7 +1,7 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
-
+const { CookieJar } = require('tough-cookie');
 const baseUrl = 'https://www.nature.com';
 
 const fixFigure = (html) => {
@@ -51,7 +51,9 @@ const getArticleList = (html) =>
 
 const getArticle = (item, ctx) =>
     ctx.cache.tryGet(item.link, async () => {
-        const response = await got(item.link);
+        const response = await got(item.link, {
+            cookieJar,
+        });
         const $ = cheerio.load(response.data);
         const responseUrl = new URL(response.url);
 
@@ -59,7 +61,7 @@ const getArticle = (item, ctx) =>
             const meta = getDataLayer($);
             item.doi = meta.content.article?.doi;
             item.author = meta.content.contentInfo.authors.join(', ');
-            item.pubDate = parseDate(meta.content.contentInfo.publishedAt * 1000) || item.pubDate;
+            item.pubDate = parseDate(meta.content.contentInfo.publishedAt, 'X') || item.pubDate;
         } else {
             const meta = JSON.parse($('script[type="application/ld+json"]').html());
             const freeAccess = meta.mainEntity.isAccessibleForFree;
@@ -69,7 +71,7 @@ const getArticle = (item, ctx) =>
                 item.doi = meta.mainEntity.sameAs.replace('https://doi.org/', '');
             }
             item.author = meta.mainEntity.author.map((author) => author.name.replace(', ', ' ')).join(', ');
-            item.category = meta.mainEntity.keywords.split(',');
+            item.category = meta.mainEntity.keywords;
             item.pubDate = parseDate(meta.mainEntity.datePublished) || item.pubDate;
 
             fixFigure($);
@@ -101,10 +103,12 @@ const getArticle = (item, ctx) =>
 const getDataLayer = (html) =>
     JSON.parse(
         html('script[data-test=dataLayer]')
-            .html()
+            .text()
             .match(/window\.dataLayer = \[(.*)\];/s)[1]
     );
 
+const cookieJar = new CookieJar();
+
 /**
  * This is generated by /nature/siteindex.debug.json
  */
@@ -144,6 +148,10 @@ const journalMap = {
             id: '43747',
             description: '43747',
         },
+        {
+            title: 'bjcreports',
+            name: 'BJC Reports',
+        },
         {
             title: 'bcj',
             name: 'Blood Cancer Journal',
@@ -234,6 +242,10 @@ const journalMap = {
             title: 'commsphys',
             name: 'Communications Physics',
         },
+        {
+            title: 'commspsychol',
+            name: 'Communications Psychology',
+        },
         {
             title: 'ejcn',
             name: 'European Journal of Clinical Nutrition',
@@ -470,12 +482,20 @@ const journalMap = {
             id: '41589',
             description: '41589',
         },
+        {
+            title: 'natchemeng',
+            name: 'Nature Chemical Engineering',
+        },
         {
             title: 'nchem',
             name: 'Nature Chemistry',
             id: '41557',
             description: '41557',
         },
+        {
+            title: 'natcities',
+            name: 'Nature Cities',
+        },
         {
             title: 'nclimate',
             name: 'Nature Climate Change',
@@ -573,6 +593,8 @@ const journalMap = {
         {
             title: 'natmentalhealth',
             name: 'Nature Mental Health',
+            id: '44220',
+            description: '44220',
         },
         {
             title: 'natmetab',
@@ -631,6 +653,8 @@ const journalMap = {
         {
             title: 'natrevbioeng',
             name: 'Nature Reviews Bioengineering',
+            id: '44222',
+            description: '44222',
         },
         {
             title: 'nrc',
@@ -672,6 +696,10 @@ const journalMap = {
             id: '43017',
             description: '43017',
         },
+        {
+            title: 'natrevelectreng',
+            name: 'Nature Reviews Electrical Engineering',
+        },
         {
             title: 'nrendo',
             name: 'Nature Reviews Endocrinology',
@@ -781,6 +809,8 @@ const journalMap = {
         {
             title: 'natwater',
             name: 'Nature Water',
+            id: '44221',
+            description: '44221',
         },
         {
             title: 'npp',
@@ -796,10 +826,18 @@ const journalMap = {
             title: 'npj2dmaterials',
             name: 'npj 2D Materials and Applications',
         },
+        {
+            title: 'npjadvmanuf',
+            name: 'npj Advanced Manufacturing',
+        },
         {
             title: 'npjamd',
             name: 'npj Aging',
         },
+        {
+            title: 'npjamar',
+            name: 'npj Antimicrobials and Resistance',
+        },
         {
             title: 'npjbiodivers',
             name: 'npj Biodiversity',
@@ -808,18 +846,42 @@ const journalMap = {
             title: 'npjbiofilms',
             name: 'npj Biofilms and Microbiomes',
         },
+        {
+            title: 'npjbiolphysmech',
+            name: 'npj Biological Physics and Mechanics',
+        },
+        {
+            title: 'npjbts',
+            name: 'npj Biological Timing and Sleep',
+        },
+        {
+            title: 'npjbiosensing',
+            name: 'npj Biosensing',
+        },
         {
             title: 'npjbcancer',
             name: 'npj Breast Cancer',
         },
+        {
+            title: 'npjcardiohealth',
+            name: 'npj Cardiovascular Health',
+        },
         {
             title: 'npjcleanwater',
             name: 'npj Clean Water',
         },
+        {
+            title: 'npjclimataction',
+            name: 'npj Climate Action',
+        },
         {
             title: 'npjclimatsci',
             name: 'npj Climate and Atmospheric Science',
         },
+        {
+            title: 'npjcomplex',
+            name: 'npj Complexity',
+        },
         {
             title: 'npjcompumats',
             name: 'npj Computational Materials',
@@ -836,18 +898,42 @@ const journalMap = {
             title: 'npjgenmed',
             name: 'npj Genomic Medicine',
         },
+        {
+            title: 'npjimaging',
+            name: 'npj Imaging',
+        },
         {
             title: 'npjmatdeg',
             name: 'npj Materials Degradation',
         },
+        {
+            title: 'npjmatsustain',
+            name: 'npj Materials Sustainability',
+        },
         {
             title: 'npjmentalhealth',
             name: 'npj Mental Health Research',
         },
+        {
+            title: 'npjmetabhealth',
+            name: 'npj Metabolic Health and Disease',
+        },
         {
             title: 'npjmgrav',
             name: 'npj Microgravity',
         },
+        {
+            title: 'npjnanophoton',
+            name: 'npj Nanophotonics',
+        },
+        {
+            title: 'npjnathazards',
+            name: 'npj Natural Hazards',
+        },
+        {
+            title: 'npjoceansustain',
+            name: 'npj Ocean Sustainability',
+        },
         {
             title: 'npjparkd',
             name: "npj Parkinson's Disease",
@@ -872,6 +958,10 @@ const journalMap = {
             title: 'npjregenmed',
             name: 'npj Regenerative Medicine',
         },
+        {
+            title: 'npjrobot',
+            name: 'npj Robotics',
+        },
         {
             title: 'npjscifood',
             name: 'npj Science of Food',
@@ -880,10 +970,26 @@ const journalMap = {
             title: 'npjscilearn',
             name: 'npj Science of Learning',
         },
+        {
+            title: 'npjspintronics',
+            name: 'npj Spintronics',
+        },
+        {
+            title: 'npjsustainagric',
+            name: 'npj Sustainable Agriculture',
+        },
+        {
+            title: 'npjsustainmobil',
+            name: 'npj Sustainable Mobility and Transport',
+        },
         {
             title: 'npjsba',
             name: 'npj Systems Biology and Applications',
         },
+        {
+            title: 'npjunconvcomput',
+            name: 'npj Unconventional Computing',
+        },
         {
             title: 'npjurbansustain',
             name: 'npj Urban Sustainability',
@@ -892,6 +998,18 @@ const journalMap = {
             title: 'npjvaccines',
             name: 'npj Vaccines',
         },
+        {
+            title: 'npjviruses',
+            name: 'npj Viruses',
+        },
+        {
+            title: 'npjwomenshealth',
+            name: "npj Women's Health",
+        },
+        {
+            title: 'dpn',
+            name: 'NPP—Digital Psychiatry and Neuroscience',
+        },
         {
             title: 'nutd',
             name: 'Nutrition & Diabetes',
@@ -973,6 +1091,7 @@ const journalMap = {
 
 module.exports = {
     baseUrl,
+    cookieJar,
     getArticle,
     getArticleList,
     getDataLayer,

package/lib/v2/ncu/jwc.js

@@ -0,0 +1,33 @@
+const got = require('@/utils/got'); // 自订的 got
+const cheerio = require('cheerio'); // 可以使用类似 jQuery 的 API HTML 解析器
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const baseUrl = 'https://jwc.ncu.edu.cn';
+    const response = await got(baseUrl);
+    const $ = cheerio.load(response.body);
+
+    const list = $('.box3 .inner ul.img-list li');
+
+    ctx.state.data = {
+        title: '南昌大学教务处',
+        link: baseUrl,
+        description: '南昌大学教务处',
+        item:
+            list &&
+            list.toArray().map((item) => {
+                const el = $(item);
+                const linkEl = el.find('a');
+                const date = el.text().split('】')[0].replace('【', '').trim();
+                const title = linkEl.attr('title');
+                const link = `${baseUrl}/${linkEl.attr('href')}`;
+                const month = date.slice(0, 2);
+
+                return {
+                    title,
+                    link,
+                    pubDate: parseDate(date, 'MM-DD').setFullYear(month < 6 ? new Date().getFullYear() - 1 : new Date().getFullYear()),
+                };
+            }),
+    };
+};

package/lib/v2/ncu/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/jwc': ['ywh555hhh'],
+};

package/lib/v2/ncu/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'ncu.edu.cn': {
+        _name: '南昌大学',
+        jwc: [
+            {
+                title: '教务通知',
+                docs: 'https://docs.rsshub.app/routes/university#nan-chang-da-xue',
+                source: ['/', '/jwtz/index.htm'],
+                target: '/ncu/jwc',
+            },
+        ],
+    },
+};

package/lib/v2/ncu/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/jwc', require('./jwc'));
+};

package/lib/v2/nhk/news.js

@@ -34,7 +34,7 @@ module.exports = async (ctx) => {
     );
 
     ctx.state.data = {
-        title: `${Object.values(meta.data.config.navigation.header).find((h) => h.keyname === 'topstories').name} | NHK WORLD-JAPAN News`,
+        title: `${Object.values(meta.data.config.navigation.header).find((h) => h.keyname === 'topstories')?.name} | NHK WORLD-JAPAN News`,
         link: `${baseUrl}/nhkworld/${lang}/news/list/`,
         item: items,
     };

package/lib/v2/nmtv/radar.js

@@ -6,7 +6,7 @@ module.exports = {
                 title: '点播',
                 docs: 'https://docs.rsshub.app/routes/traditional-media#nei-meng-gu-guang-bo-dian-shi-tai-dian-bo',
                 source: ['/'],
-                target: (params, url) => `/nmtv/column/${new URL(url).toString.split(/\/folder/).pop()}`,
+                target: (params, url) => `/nmtv/column/${url.split(/\/folder/).pop()}`,
             },
         ],
     },

package/lib/v2/nytimes/radar.js

@@ -14,12 +14,6 @@ module.exports = {
                 source: ['/books/best-sellers/:category', '/books/best-sellers/'],
                 target: '/nytimes/book/:category',
             },
-            {
-                title: '作者新闻',
-                docs: 'https://docs.rsshub.app/routes/traditional-media#niu-yue-shi-bao',
-                source: '/by/:byline',
-                target: '/nytimes/author/:byline',
-            },
             {
                 title: '新闻',
                 docs: 'https://docs.rsshub.app/routes/traditional-media#niu-yue-shi-bao',

package/lib/v2/nytimes/router.js

@@ -1,5 +1,4 @@
 module.exports = (router) => {
-    router.get('/author/:byline', require('./author.js'));
     router.get('/book/:category?', require('./book.js'));
     router.get('/daily_briefing_chinese', require('./daily_briefing_chinese'));
     router.get('/:lang?', require('./index'));

package/lib/v2/oo-software/radar.js

@@ -6,7 +6,7 @@ module.exports = {
                 title: 'Changelog',
                 docs: 'https://docs.rsshub.app/routes/program-update#oo-software-changelog',
                 source: ['/en/changelog'],
-                target: (params, url) => `/oo-software/changelog/${new URL(url).match(/\/en\/(.*?)\/changelog/)[1]}`,
+                target: (params, url) => `/oo-software/changelog/${url.match(/\/en\/(.*?)\/changelog/)[1]}`,
             },
         ],
     },

package/lib/v2/patagonia/maintainer.js

@@ -1,3 +1,3 @@
 module.exports = {
-    '/patagonia/new-arrivals/:category': ['NavePnow'],
+    '/patagonia/new-arrivals/:category': ['EthanWng97'],
 };

package/lib/v2/phoronix/index.js

@@ -1,26 +1,132 @@
 const parser = require('@/utils/rss-parser');
 const cheerio = require('cheerio');
 const got = require('@/utils/got');
+const dayjs = require('dayjs');
+dayjs.extend(require('dayjs/plugin/utc'));
+dayjs.extend(require('dayjs/plugin/timezone'));
+
+const redirectCacheKey = 'phoronix:redirect';
+const webArticlesCacheKey = 'phoronix:web-articles';
+const articleCacheKey = 'phoronix:articles';
 
 const baseUrl = 'https://www.phoronix.com';
+const rssUrl = `${baseUrl}/rss.php`;
 
-module.exports = async (ctx) => {
-    const { page, queryOrItem } = ctx.params;
-    const rssUrl = new URL('/rss.php', baseUrl);
-    rssUrl.searchParams.set('page', page);
+const feedFetch = async () => {
+    const feed = await parser.parseURL(rssUrl);
+    return {
+        title: feed.title,
+        link: feed.link,
+        description: feed.description,
+        item: feed.items,
+        language: feed.language,
+        icon: 'https://www.phoronix.com/android-chrome-192x192.png',
+        image: 'https://www.phoronix.com/android-chrome-192x192.png',
+        logo: 'https://www.phoronix.com/phxcms7-css/phoronix.png',
+        // Copied from the web page metadata
+        category: [
+            'Linux Hardware Reviews',
+            'Linux hardware benchmarks',
+            'Linux Hardware',
+            'Linux benchmarking',
+            'Desktop Linux',
+            'GNU/Linux benchmarks',
+            'Open Source AMD',
+            'Linux How To',
+            'X.Org drivers',
+            'Ubuntu hardware',
+            'Phoronix Test Suite',
+        ],
+    };
+};
+
+const webFetchCb = (response) => {
+    const $ = cheerio.load(response.body);
+    return {
+        title: $('title').text(),
+        link: response.url,
+        description: $('meta[name="Description"]').attr('content'),
+        item: [
+            ...new Set(
+                $('#main a')
+                    .toArray()
+                    .map((e) => e.attribs.href)
+            ),
+        ]
+            .filter((link) => link && (link.startsWith('/review/') || link.startsWith('/news/')))
+            .map((link) => ({ link: `${baseUrl}${link}` })),
+        language: 'en-us',
+        icon: 'https://www.phoronix.com/android-chrome-192x192.png',
+        image: 'https://www.phoronix.com/android-chrome-192x192.png',
+        logo: 'https://www.phoronix.com/phxcms7-css/phoronix.png',
+        category: $('meta[name="keywords"]').attr('content').split(', '),
+    };
+};
+
+const webFetch = (ctx, url) =>
+    ctx.cache.tryGet(`${webArticlesCacheKey}:${url}`, async () => {
+        try {
+            return webFetchCb(await got(url));
+        } catch (error) {
+            if (error.name === 'HTTPError' && error.response.statusCode === 404) {
+                return '404';
+            }
+            throw error;
+        }
+    });
+
+const legacyFetch = async (ctx, page, queryOrItem) => {
+    const legacyUrl = new URL('/scan.php', baseUrl);
+    legacyUrl.searchParams.set('page', page);
     if (queryOrItem) {
         if (page === 'category') {
-            rssUrl.searchParams.set('item', queryOrItem);
+            legacyUrl.searchParams.set('item', queryOrItem);
         } else {
-            rssUrl.searchParams.set('q', queryOrItem);
+            legacyUrl.searchParams.set('q', queryOrItem);
         }
     }
 
-    const feed = await parser.parseURL(rssUrl.toString());
+    let response;
+    const webUrl = await ctx.cache.tryGet(`${redirectCacheKey}:${legacyUrl.toString()}`, async () => {
+        response = await got(legacyUrl.toString());
+        return response.url;
+    });
+    if (response) {
+        const feed = webFetchCb(response);
+        ctx.cache.set(`${webArticlesCacheKey}:${webUrl}`, feed);
+        return feed;
+    }
+    return await webFetch(ctx, webUrl);
+};
 
-    const items = await Promise.all(
-        feed.items.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
+const tryFetch = async (ctx, category, topic) => {
+    const webUrl = topic ? `${baseUrl}/${category}/${topic}` : `${baseUrl}/${category}`;
+    let feed = await webFetch(ctx, webUrl);
+    if (feed === '404') {
+        feed = await legacyFetch(ctx, category, topic);
+    }
+    return feed;
+};
+
+module.exports = async (ctx) => {
+    const { category, topic } = ctx.params;
+    let feed;
+    switch (category) {
+        case 'category':
+        case 'news_topic':
+            feed = await legacyFetch(ctx, category, topic);
+            break;
+        case 'rss':
+            feed = await feedFetch();
+            break;
+        default:
+            feed = category ? await tryFetch(ctx, category, topic) : await feedFetch();
+            break;
+    }
+
+    feed.item = await Promise.all(
+        feed.item.map((item) =>
+            ctx.cache.tryGet(`${articleCacheKey}:${item.link}`, async () => {
                 const response = await got(item.link);
                 const html = response.body;
                 const $ = cheerio.load(html);
@@ -28,11 +134,52 @@ module.exports = async (ctx) => {
 
                 // Author
                 const authorSelector = $('.author > a');
-                // thel last 2 are the category and comments
+                // the last 2 are the category and comments
                 const author = authorSelector
                     .slice(0, authorSelector.length - 2)
                     .toArray()
                     .map((e) => $(e).text());
+                const category = [];
+                if (item.link.includes('/news/')) {
+                    category.push('News');
+                } else if (item.link.includes('/review/')) {
+                    category.push('Review');
+                }
+                const categorySelector = authorSelector.eq(-2);
+                if (categorySelector.length) {
+                    category.push(categorySelector.text());
+                }
+                let pubDate;
+                if (!item.pubDate) {
+                    // the text next to the category is the date
+                    let pubDateReadable = categorySelector.length && categorySelector[0].nextSibling?.nodeValue;
+                    if (pubDateReadable) {
+                        pubDateReadable = pubDateReadable.replace(/on|at|\./g, '').trim();
+                        if (/\d{4}$/.test(pubDateReadable)) {
+                            // Only date, no time
+                            // Michael Larabel lives in Indiana, USA, so we assume TZ=America/Indiana/Indianapolis
+                            // https://www.phoronix.com/review/phoronix_office_2014
+                            // Here we use the trick to take daylight saving into account.
+                            pubDate = dayjs
+                                // If we don't append "UTC" at the end,
+                                // dayjs.utc() may still parse the date in the platform (local) timezone.
+                                // E.g., if the platform timezone is UTC+8, then:
+                                // > dayjs.utc('2 Dec 2023').toString()
+                                // 'Fri, 01 Dec 2023 16:00:00 GMT'
+                                // > dayjs.utc('2 Dec 2023 UTC').toString()
+                                // 'Sat, 02 Dec 2023 00:00:00 GMT'
+                                // Append "UTC" at the end to explicitly prohibit the weird behavior.
+                                .utc(`${pubDateReadable} 08:00 UTC`)
+                                .tz('America/Indiana/Indianapolis', true);
+                        } else {
+                            // date, time, and timezone (including daylight saving)
+                            pubDate = dayjs(pubDateReadable);
+                        }
+                        if (!pubDate.isValid()) {
+                            pubDate = pubDateReadable;
+                        }
+                    }
+                }
 
                 // Maybe it's paginated
                 const links = $('.pagination > a')
@@ -55,53 +202,35 @@ module.exports = async (ctx) => {
                     content.append(pages);
                 }
 
-                // Summary
-                const summary = $('.content > p:nth-child(1)');
-
-                // High res images
-                content.find('img').each((_, img) => {
-                    if (img.attribs.src.endsWith('_med')) {
-                        img.attribs.src = img.attribs.src.replace('_med', '_show');
+                const images = content.find('img');
+                // Remove topic image
+                const topicImage = images.first();
+                if (topicImage.attr('src')?.startsWith('/assets/categories/')) {
+                    const topicImageContainer = topicImage.parent();
+                    if (!topicImageContainer.text().trim()) {
+                        topicImageContainer.remove();
+                    } else {
+                        topicImage.remove();
                     }
+                }
+                // High-res images
+                images.each((_, img) => {
+                    img.attribs.src = img.attribs.src.replace(/_med$/, '');
                 });
 
                 return {
-                    title: item.title,
-                    id: item.guid,
-                    pubDate: item.pubDate,
+                    title: item.title || $('article h1').text(),
+                    pubDate: item.pubDate || pubDate,
                     author: author.join(', '),
                     link: item.link,
-                    summary: summary.html(),
+                    summary: $('meta[name="twitter:description"]').attr('content'),
                     description: content.html(),
-                    icon: 'https://www.phoronix.com/android-chrome-192x192.png',
-                    logo: 'https://www.phoronix.com/phxcms7-css/phoronix.png',
+                    image: $('meta[name="twitter:image"]').attr('content'),
+                    category: item.category || category,
                 };
             })
         )
     );
 
-    ctx.state.data = {
-        title: feed.title,
-        link: feed.link,
-        description: feed.description,
-        item: items,
-        language: feed.language,
-        icon: 'https://www.phoronix.com/android-chrome-192x192.png',
-        image: 'https://www.phoronix.com/android-chrome-192x192.png',
-        logo: 'https://www.phoronix.com/phxcms7-css/phoronix.png',
-        // Copied from thier web page metadata
-        category: [
-            'Linux Hardware Reviews',
-            'Linux hardware benchmarks',
-            'Linux Hardware',
-            'Linux benchmarking',
-            'Desktop Linux',
-            'GNU/Linux benchmarks',
-            'Open Source AMD',
-            'Linux How To',
-            'X.Org drivers',
-            'Ubuntu hardware',
-            'Phoronix Test Suite',
-        ],
-    };
+    ctx.state.data = feed;
 };