rsshub 1.0.0-master.fc015b9 → 1.0.0-master.fc1b99f

package/lib/routes/natgeo/natgeo.ts

@@ -0,0 +1,82 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import { load } from 'cheerio';
+import got from '@/utils/got';
+import { parseDate } from '@/utils/parse-date';
+
+// https://www.natgeomedia.com//article/
+
+async function loadContent(link) {
+    const { data } = await got(link);
+    const $ = load(data);
+    const dtStr = $('.content-title-area')
+        .find('h6')
+        .first()
+        .html()
+        .replaceAll(/ /gi, ' ')
+        .trim();
+
+    let description = $('article').first().html() + $('article').eq(1).html();
+    if (/photo/.test(link)) {
+        description = $('#content-album').html() + description;
+    }
+    return {
+        title: $('title').text(),
+        pubDate: parseDate(dtStr, 'MMM. DD YYYY'),
+        description,
+    };
+}
+
+export const route: Route = {
+    path: '/:cat/:type?',
+    categories: ['travel'],
+    example: '/natgeo/environment/article',
+    parameters: { cat: '分类', type: '类型, 例如`https://www.natgeomedia.com/environment/photo/`对应 `cat`, `type` 分别为 `environment`, `photo`' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['natgeomedia.com/:cat/:type', 'natgeomedia.com/'],
+            target: '/:cat/:type',
+        },
+    ],
+    name: '分类',
+    maintainers: ['fengkx'],
+    handler,
+};
+
+async function handler(ctx) {
+    const type = ctx.req.param('type') ?? '';
+    const url = `https://www.natgeomedia.com/${ctx.req.param('cat')}/${type}`;
+    const res = await got(url);
+    const $ = load(res.data);
+
+    const urlList = $('.article-link-w100')
+        .find('.read-btn')
+        .toArray()
+        .map((i) => ({
+            link: $(i).find('a[href]').first().attr('href'),
+        }));
+
+    const out = await Promise.all(
+        urlList.map(async (i) => {
+            const link = i.link;
+            const single = {
+                link,
+            };
+            const other = await cache.tryGet(link, () => loadContent(link));
+            return { ...single, ...other };
+        })
+    );
+    return {
+        title: $('title').text(),
+        link: url,
+        item: out,
+    };
+}

package/lib/routes/nationalgeographic/latest-stories.ts

@@ -0,0 +1,85 @@
+import { Route } from '@/types';
+import { getCurrentPath } from '@/utils/helpers';
+const __dirname = getCurrentPath(import.meta.url);
+
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import { art } from '@/utils/render';
+import * as path from 'node:path';
+
+const findNatgeo = ($) =>
+    JSON.parse(
+        $('script')
+            .text()
+            .match(/\['__natgeo__']=({.*?});/)[1]
+    );
+
+export const route: Route = {
+    path: '/latest-stories',
+    categories: ['travel'],
+    example: '/nationalgeographic/latest-stories',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['www.nationalgeographic.com/pages/topic/latest-stories'],
+        },
+    ],
+    name: 'Latest Stories',
+    maintainers: ['miles170'],
+    handler,
+    url: 'www.nationalgeographic.com/pages/topic/latest-stories',
+};
+
+async function handler() {
+    const currentUrl = 'https://www.nationalgeographic.com/pages/topic/latest-stories';
+    const response = await got(currentUrl);
+    const $ = load(response.data);
+    const items = await Promise.all(
+        findNatgeo($)
+            .page.content.hub.frms.flatMap((e) => e.mods)
+            .flatMap((m) => m.tiles?.filter((t) => t.ctas[0]?.text === 'natgeo.ctaText.read'))
+            .filter(Boolean)
+            .map((i) => ({
+                title: i.title,
+                link: i.ctas[0].url,
+                category: i.tags.map((t) => t.name),
+            }))
+            .map((item) =>
+                cache.tryGet(item.link, async () => {
+                    const response = await got(item.link);
+                    const $ = load(response.data);
+                    const mods = findNatgeo($).page.content.article.frms.find((f) => f.cmsType === 'ArticleBodyFrame').mods;
+                    const bodyTile = mods.find((m) => m.edgs[0].cmsType === 'ArticleBodyTile').edgs[0];
+
+                    item.author = bodyTile.cntrbGrp
+                        .flatMap((c) => c.contributors)
+                        .map((c) => c.displayName)
+                        .join(', ');
+                    item.description = art(path.join(__dirname, 'templates/stories.art'), {
+                        ldMda: bodyTile.ldMda,
+                        description: bodyTile.dscrptn,
+                        body: bodyTile.bdy,
+                    });
+                    item.pubDate = parseDate(bodyTile.pbDt);
+
+                    return item;
+                })
+            )
+    );
+
+    return {
+        title: $('meta[property="og:title"]').attr('content'),
+        link: currentUrl,
+        item: items.filter((item) => item !== null),
+    };
+}

package/lib/routes/nationalgeographic/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'National Geographic',
+    url: 'www.nationalgeographic.com',
+};

package/lib/routes/nationalgeographic/templates/stories.art

@@ -0,0 +1,44 @@
+{{ if ldMda }}
+    <figure>
+        <img src="{{ ldMda.image.src }}" alt="{{ ldMda.image.altText }}">
+        <figcaption>{{ ldMda.caption }}</figcaption>
+    </figure>
+{{ /if }}
+
+{{ if description }}
+    <p><b>{{ description }}</b></p>
+{{ /if }}
+
+{{ each body b }}
+    {{ if b.type === 'p' }}
+        <p>{{@ b.cntnt.mrkup }}</p>
+    {{ else if b.type === 'inline' }}
+        {{ if b.cntnt.cmsType === 'image' }}
+            <figure>
+                <img src="{{ b.cntnt.image.src }}" alt="{{ b.cntnt.image.altText }}">
+                <figcaption>{{@ b.cntnt.caption }}</figcaption>
+            </figure>
+        {{ else if b.cntnt.cmsType === 'imagegroup' }}
+            {{ each b.cntnt.images img }}
+                <figure>
+                    <img src="{{ img.image.src }}" alt="{{ img.image.altText }}">
+                    <figcaption>{{@ img.caption }}</figcaption>
+                </figure>
+            {{ /each }}
+        {{ else if b.cntnt.cmsType === 'editorsNote' }}
+            <p>{{@ b.cntnt.note }}</p>
+        {{ else if b.cntnt.cmsType === 'listicle' }}
+            <p>{{@ b.cntnt.text }}</p>
+        {{ else if b.cntnt.cmsType === 'pullquote' }}
+            <backquote>{{@ b.cntnt.quote }}</backquote>
+        {{ else if b.cntnt.cmsType === 'source' }}
+            <a href="{{ b.cntnt.src }}">{{ b.cntnt.src }}</a>
+        {{ else if b.cntnt.cmsType === 'video' }}
+            <figure>
+                <img src="{{ b.cntnt.image.src }}" alt="{{ b.cntnt.image.altText }}">
+                <figcaption>{{@ b.cntnt.title }}</figcaption>
+                <figcaption>{{@ b.cntnt.description }}</figcaption>
+            </figure>
+        {{ /if }}
+    {{ /if }}
+{{ /each }}

package/lib/routes/nature/cover.ts

@@ -0,0 +1,94 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+// The content is generateed by undocumentated API of nature journals
+// This router has **just** been tested in:
+// nature:           Nature
+// nbt:              Nature Biotechnology
+// neuro:            Nature Neuroscience
+// ng:               Nature Genetics
+// ni:               Nature Immunology
+// nmeth:            Nature Method
+// nchem:            Nature Chemistry
+// nmat:             Nature Materials
+// natmachintell:    Nature Machine Intelligence
+// ncb:              Nature Cell Biology
+// nplants:          Nature Plants
+// natastron:        Nature Astronomy
+// nphys             Nature Physics
+
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import { baseUrl, journalMap } from './utils';
+import { CookieJar } from 'tough-cookie';
+
+export const route: Route = {
+    path: '/cover',
+    categories: ['journal'],
+    example: '/nature/cover',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['nature.com/'],
+        },
+    ],
+    name: 'Cover Story',
+    maintainers: ['y9c'],
+    handler,
+    url: 'nature.com/',
+    description: `Subscribe to the cover images of the Nature journals, and get the latest publication updates in time.`,
+};
+
+async function handler() {
+    const cookieJar = new CookieJar();
+
+    await got('https://www.nature.com', { cookieJar });
+
+    const journals = journalMap.items
+        .filter((j) => j.id)
+        .map((j) => ({
+            ...j,
+            link: `${baseUrl}/${j.title}/current-issue`,
+        }));
+
+    const out = await Promise.all(
+        journals.map((journal) =>
+            cache.tryGet(journal.link, async () => {
+                const { id, name, link } = journal;
+
+                const response = await got(link, { cookieJar });
+
+                const capturingRegex = /volumes\/(?<volume>\d+)\/issues\/(?<issue>\d+)/;
+                const { volume, issue } = response.url.match(capturingRegex).groups;
+                const imageUrl = `https://media.springernature.com/full/springer-static/cover-hires/journal/${id}/${volume}/${issue}?as=webp`;
+                const contents = `<div align="center"><img src="${imageUrl}" alt="Volume ${volume} Issue ${issue}"></div>`;
+
+                const $ = load(response.data);
+                const date = $('title').text().split(',')[1].trim();
+                const issueDescription = $('div[data-test=issue-description]').html() ?? '';
+
+                return {
+                    title: `${name} | Volume ${volume} Issue ${issue}`,
+                    description: contents + issueDescription,
+                    link: response.url,
+                    pubDate: parseDate(date, 'MMMM YYYY'),
+                };
+            })
+        )
+    );
+
+    return {
+        title: 'Nature Covers Story',
+        description: 'Find out the cover story of some Nature journals.',
+        link: baseUrl,
+        item: out,
+    };
+}

package/lib/routes/nature/highlight.ts

@@ -0,0 +1,50 @@
+import { Route } from '@/types';
+import { load } from 'cheerio';
+import got from '@/utils/got';
+import { baseUrl, cookieJar, getArticleList, getArticle } from './utils';
+
+export const route: Route = {
+    path: '/highlight/:journal?',
+    categories: ['journal'],
+    example: '/nature/highlight',
+    parameters: { journal: 'short name for a journal, `nature` by default' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: true,
+    },
+    radar: [
+        {
+            source: ['nature.com/:journal/articles', 'nature.com/:journal', 'nature.com/'],
+            target: '/highlight/:journal',
+        },
+    ],
+    name: 'Research Highlight',
+    maintainers: [],
+    handler,
+    description: `:::warning
+  Only some journals are supported.
+  :::`,
+};
+
+async function handler(ctx) {
+    const { journal = 'nature' } = ctx.req.param();
+    const url = `${baseUrl}/${journal}/articles?type=research-highlight`;
+
+    const res = await got(url, { cookieJar });
+    const $ = load(res.data);
+
+    let items = getArticleList($);
+
+    items = await Promise.all(items.map((item) => getArticle(item)));
+
+    return {
+        title: $('title').text().trim(),
+        description: $('meta[name=description]').attr('content'),
+        link: url,
+        item: items,
+    };
+}

package/lib/routes/nature/namespace.ts

@@ -0,0 +1,9 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Nature Journal',
+    url: 'nature.com',
+    description: `:::tip
+You can get all short name of a journal from [https://www.nature.com/siteindex](https://www.nature.com/siteindex) or [Journal List](#nature-journal-journal-list).
+:::`,
+};

package/lib/routes/nature/news-and-comment.ts

@@ -0,0 +1,52 @@
+import { Route } from '@/types';
+// example usage: `/nature/news-and-comment/ng`
+// The journals from NPG are run by different group of people,
+// and the website of may not be consitent for all the journals
+//
+// This router has **just** been tested in:
+// nbt:              Nature Biotechnology
+// neuro:            Nature Neuroscience
+// ng:               Nature Genetics
+// ni:               Nature Immunology
+// nmeth:            Nature Method
+// nchem:            Nature Chemistry
+// nmat:             Nature Materials
+// natmachintell:    Nature Machine Intelligence
+
+import { load } from 'cheerio';
+import got from '@/utils/got';
+import { baseUrl, cookieJar, getArticleList, getArticle } from './utils';
+
+export const route: Route = {
+    path: '/news-and-comment/:journal?',
+    radar: [
+        {
+            source: ['nature.com/latest-news', 'nature.com/news', 'nature.com/'],
+            target: '/news',
+        },
+    ],
+    name: 'Unknown',
+    maintainers: ['y9c', 'TonyRL'],
+    handler,
+    url: 'nature.com/latest-news',
+};
+
+async function handler(ctx) {
+    const journal = ctx.req.param('journal');
+    const pageURL = `${baseUrl}/${journal}/news-and-comment`;
+
+    const pageResponse = await got(pageURL, { cookieJar });
+    const pageCapture = load(pageResponse.data);
+    const pageDescription = pageCapture('meta[name=description]').attr('content') || 'Nature, a nature research journal';
+
+    let items = getArticleList(pageCapture);
+
+    items = await Promise.all(items.map((item) => getArticle(item)));
+
+    return {
+        title: pageCapture('title').text(),
+        description: pageDescription,
+        link: pageURL,
+        item: items,
+    };
+}

package/lib/routes/nature/news.ts

@@ -0,0 +1,56 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import { load } from 'cheerio';
+import got from '@/utils/got';
+import { parseDate } from '@/utils/parse-date';
+import { baseUrl, cookieJar, getArticle } from './utils';
+
+export const route: Route = {
+    path: '/news',
+    categories: ['journal'],
+    example: '/nature/news',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: true,
+    },
+    radar: [
+        {
+            source: ['nature.com/latest-news', 'nature.com/news', 'nature.com/'],
+        },
+    ],
+    name: 'Nature News',
+    maintainers: ['y9c', 'TonyRL'],
+    handler,
+    url: 'nature.com/latest-news',
+};
+
+async function handler() {
+    const url = `${baseUrl}/latest-news`;
+    const res = await got(url, { cookieJar });
+    const $ = load(res.data);
+
+    let items = $('.c-article-item__content')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('h3').text(),
+                link: baseUrl + item.find('a').attr('href'),
+                pubDate: parseDate(item.find('.c-article-item__date').text()),
+            };
+        });
+
+    items = await Promise.all(items.map((item) => cache.tryGet(item.link, () => getArticle(item))));
+
+    return {
+        title: 'Nature | Latest News',
+        description: $('meta[name=description]').attr('content'),
+        link: url,
+        item: items,
+    };
+}

package/lib/routes/nature/research.ts

@@ -0,0 +1,80 @@
+import { Route } from '@/types';
+// example usage: `/nature/research/ng`
+// The journals from NPG are run by different group of people,
+// and the website of may not be consitent for all the journals
+//
+// This router has **just** been tested in:
+// nature:           Nature
+// nbt:              Nature Biotechnology
+// neuro:            Nature Neuroscience
+// ng:               Nature Genetics
+// ni:               Nature Immunology
+// nmeth:            Nature Method
+// nchem:            Nature Chemistry
+// nmat:             Nature Materials
+// natmachintell:    Nature Machine Intelligence
+
+import { load } from 'cheerio';
+import got from '@/utils/got';
+import { baseUrl, cookieJar, getArticleList, getDataLayer, getArticle } from './utils';
+
+export const route: Route = {
+    path: '/research/:journal?',
+    categories: ['journal'],
+    example: '/nature/research/ng',
+    parameters: { journal: 'short name for a journal, `nature` by default' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: true,
+    },
+    radar: [
+        {
+            source: ['nature.com/:journal/research-articles', 'nature.com/:journal', 'nature.com/'],
+            target: '/research/:journal',
+        },
+    ],
+    name: 'Latest Research',
+    maintainers: ['y9c', 'TonyRL'],
+    handler,
+    description: `|   \`:journal\`  |   Full Name of the Journal  | Route                                                                              |
+  | :-----------: | :-------------------------: | ---------------------------------------------------------------------------------- |
+  |     nature    |            Nature           | [/nature/research/nature](https://rsshub.app/nature/research/nature)               |
+  |      nbt      |     Nature Biotechnology    | [/nature/research/nbt](https://rsshub.app/nature/research/nbt)                     |
+  |     neuro     |     Nature Neuroscience     | [/nature/research/neuro](https://rsshub.app/nature/research/neuro)                 |
+  |       ng      |       Nature Genetics       | [/nature/research/ng](https://rsshub.app/nature/research/ng)                       |
+  |       ni      |      Nature Immunology      | [/nature/research/ni](https://rsshub.app/nature/research/ni)                       |
+  |     nmeth     |        Nature Method        | [/nature/research/nmeth](https://rsshub.app/nature/research/nmeth)                 |
+  |     nchem     |       Nature Chemistry      | [/nature/research/nchem](https://rsshub.app/nature/research/nchem)                 |
+  |      nmat     |       Nature Materials      | [/nature/research/nmat](https://rsshub.app/nature/research/nmat)                   |
+  | natmachintell | Nature Machine Intelligence | [/nature/research/natmachintell](https://rsshub.app/nature/research/natmachintell) |
+
+  -   Using router (\`/nature/research/\` + "short name for a journal") to query latest research paper for a certain journal of Nature Publishing Group.
+      If the \`:journal\` parameter is blank, then latest research of Nature will return.
+  -   The journals from NPG are run by different group of people, and the website of may not be consitent for all the journals
+  -   Only abstract is rendered in some researches`,
+};
+
+async function handler(ctx) {
+    const journal = ctx.req.param('journal') ?? 'nature';
+    const pageURL = `${baseUrl}/${journal}/research-articles`;
+
+    const pageResponse = await got(pageURL, { cookieJar });
+    const pageCapture = load(pageResponse.data);
+
+    const pageTitle = getDataLayer(pageCapture).content.journal.title;
+
+    let items = getArticleList(pageCapture);
+
+    items = await Promise.all(items.map((item) => getArticle(item)));
+
+    return {
+        title: `Nature (${pageTitle}) | Latest Research`,
+        description: pageCapture('meta[name="description"]').attr('content') || `Nature, a nature research journal`,
+        link: pageURL,
+        item: items,
+    };
+}

package/lib/routes/nature/siteindex.ts

@@ -0,0 +1,68 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { baseUrl, cookieJar } from './utils';
+
+export const route: Route = {
+    path: '/siteindex',
+    categories: ['journal'],
+    example: '/nature/siteindex',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Journal List',
+    maintainers: ['TonyRL'],
+    handler,
+};
+
+async function handler(ctx) {
+    const response = await got(`${baseUrl}/siteindex`, { cookieJar });
+    const $ = load(response.data);
+
+    let items = $('li[class^="grid mq640-grid-12"]')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('a').attr('href').replaceAll('/', ''),
+                name: item.find('a').text(),
+                link: baseUrl + item.find('a').attr('href'),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            cache.tryGet(`nature:siteindex:${item.title}`, async () => {
+                const response = await got(item.link, { cookieJar });
+                const $ = load(response.data);
+
+                delete item.link;
+                try {
+                    item.id = $('.app-latest-issue-row__image img')
+                        .attr('src')
+                        .match(/.*\/journal\/(\d{5})/)[1];
+                    item.description = item.id;
+                } catch {
+                    //
+                }
+                return item;
+            })
+        )
+    );
+
+    return {
+        title: 'Nature siteindex',
+        link: response.url,
+        item: items,
+    };
+    ctx.set('json', {
+        items,
+    });
+}