rsshub 1.0.0-master.f71451d → 1.0.0-master.f72af1b

package/lib/routes/gisreportsonline/index.ts

@@ -0,0 +1,60 @@
+import { Route } from '@/types';
+
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+
+export const route: Route = {
+    path: '/:path{.*}',
+    categories: ['new-media'],
+    example: '/gis/c/security-challenges/',
+    parameters: { path: '包含"Reports"页面下的路径' },
+    name: '报告',
+    maintainers: ['dzx-dzx'],
+    radar: [
+        {
+            source: ['www.gisreportsonline.com'],
+        },
+    ],
+    handler,
+};
+
+async function handler(ctx) {
+    const rootUrl = 'https://www.gisreportsonline.com';
+    const currentUrl = `${rootUrl}/${ctx.req.param('path')}`;
+    const response = await ofetch(currentUrl);
+
+    const $ = load(response);
+
+    const list = $('article h3 a')
+        .toArray()
+        .map((e) => ({ link: $(e).attr('href'), title: $(e).text() }));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const content = load(await ofetch(item.link));
+                const ldjson = JSON.parse(content('script.rank-math-schema-pro').text())['@graph'].find((e) => e['@type'] === 'NewsArticle');
+
+                item.pubDate = parseDate(ldjson.datePublished);
+                item.updated = parseDate(ldjson.dateModified);
+                item.author = [ldjson.author];
+                item.category = ldjson.keywords.split(',');
+                item.language = ldjson.inLanguage;
+
+                item.description = content('header.entry-header ~ :not(#pos-conclusion ~ *)')
+                    .toArray()
+                    .map((e) => content(e).prop('outerHTML'))
+                    .join('');
+
+                return item;
+            })
+        )
+    );
+    return {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+}

package/lib/routes/gisreportsonline/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'GIS Reports',
+    url: 'www.gisreportsonline.com',
+};

package/lib/routes/github/activity.ts

@@ -0,0 +1,56 @@
+import { Route, ViewType } from '@/types';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+import Parser from 'rss-parser';
+import sanitizeHtml from 'sanitize-html';
+
+const parser = new Parser();
+
+export const route: Route = {
+    path: '/activity/:user',
+    name: 'User Activities',
+    maintainers: ['hyoban'],
+    example: '/github/activity/DIYgod',
+    categories: ['programming', 'popular'],
+    view: ViewType.Articles,
+    parameters: {
+        user: 'GitHub username',
+    },
+    description: 'Get the activities of a user on GitHub, based on the GitHub official RSS feed',
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['github.com/:user'],
+            target: '/activity/:user',
+        },
+    ],
+    handler: async (ctx) => {
+        const { user } = ctx.req.param();
+        const response = (await ofetch(`https://github.com/${user}.atom`)) as Blob;
+        const raw = await response.text();
+        // <media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/8266075?s=30&amp;v=4"/>
+        const image = raw.match(/<media:thumbnail height="30" width="30" url="(.+?)"/)?.[1];
+        const feed = await parser.parseString(raw);
+        return {
+            title: `${user}'s GitHub activities`,
+            link: feed.link,
+            image,
+            item: feed.items.map((item) => ({
+                title: item.title ?? '',
+                link: item.link,
+                description: sanitizeHtml(item.content?.replace(/href="\/(.+?)"/g, `href="https://github.com/$1"`) ?? '', { allowedTags: [...sanitizeHtml.defaults.allowedTags, 'img'] }),
+                pubDate: item.pubDate ? parseDate(item.pubDate) : undefined,
+                author: item.author,
+                guid: item.id,
+                image,
+            })),
+        };
+    },
+};

package/lib/routes/github/comments.ts

@@ -1,5 +1,5 @@
 import { Route } from '@/types';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { parseDate } from '@/utils/parse-date';
 import MarkdownIt from 'markdown-it';
 const md = MarkdownIt({
@@ -59,16 +59,16 @@ async function handler(ctx) {
 }
 
 async function allIssues(ctx, user, repo, limit, headers) {
-    const response = await got(`${apiUrl}/repos/${user}/${repo}/issues/comments`, {
+    const response = await ofetch.raw(`${apiUrl}/repos/${user}/${repo}/issues/comments`, {
         headers,
-        searchParams: {
+        query: {
             sort: 'updated',
             direction: 'desc',
             per_page: limit,
         },
     });
 
-    const timeline = response.data;
+    const timeline = response._data;
 
     const items = timeline.map((item) => {
         const actor = item.actor?.login ?? item.user?.login ?? 'ghost';
@@ -86,14 +86,13 @@ async function allIssues(ctx, user, repo, limit, headers) {
         };
     });
 
-    // response headers is broken due to #14922
-    // const rateLimit = {
-    //     limit: Number.parseInt(response.headers['x-ratelimit-limit']),
-    //     remaining: Number.parseInt(response.headers['x-ratelimit-remaining']),
-    //     reset: parseDate(Number.parseInt(response.headers['x-ratelimit-reset']) * 1000),
-    //     resoure: response.headers['x-ratelimit-resource'],
-    //     used: Number.parseInt(response.headers['x-ratelimit-used']),
-    // };
+    const rateLimit = {
+        limit: Number.parseInt(response.headers.get('x-ratelimit-limit')),
+        remaining: Number.parseInt(response.headers.get('x-ratelimit-remaining')),
+        reset: parseDate(Number.parseInt(response.headers.get('x-ratelimit-reset')) * 1000),
+        resoure: response.headers.get('x-ratelimit-resource'),
+        used: Number.parseInt(response.headers.get('x-ratelimit-used')),
+    };
 
     const ret = {
         title: `${user}/${repo}: Issue & Pull request comments`,
@@ -103,35 +102,38 @@ async function allIssues(ctx, user, repo, limit, headers) {
 
     ctx.set('json', {
         ...ret,
-        // rateLimit,
+        rateLimit,
     });
     return ret;
 }
 
 async function singleIssue(ctx, user, repo, number, limit, headers) {
-    const response = await got(`${apiUrl}/repos/${user}/${repo}/issues/${number}`, {
+    const response = await ofetch.raw(`${apiUrl}/repos/${user}/${repo}/issues/${number}`, {
         headers,
     });
-    const issue = response.data;
+    const issue = response._data;
     const type = issue.pull_request ? 'pull' : 'issue';
 
-    const timelineResponse = await got(issue.timeline_url, {
+    let timelineResponse = await ofetch.raw(issue.timeline_url, {
         headers,
-        searchParams: {
+        query: {
             per_page: limit,
         },
     });
-    const timeline = timelineResponse.data;
-
-    const items = [
-        {
+    const items = [];
+    const lastUrl = timelineResponse.headers.get('link')?.match(/<(\S+?)>; rel="last"/)?.[1];
+    if (lastUrl) {
+        timelineResponse = await ofetch.raw(lastUrl, { headers });
+    } else {
+        items.push({
             title: `${issue.user.login} created ${user}/${repo}: ${typeDict[type].title} #${issue.number}`,
             description: issue.body ? md.render(issue.body) : null,
             author: issue.user.login,
             pubDate: parseDate(issue.created_at),
             link: `${issue.html_url}#issue-${issue.id}`,
-        },
-    ];
+        });
+    }
+    const timeline = timelineResponse._data;
 
     for (const item of timeline) {
         const actor = item.actor?.login ?? item.user?.login ?? 'ghost';
@@ -194,13 +196,13 @@ async function singleIssue(ctx, user, repo, number, limit, headers) {
 
     ctx.set('json', {
         ...ret,
-        // rateLimit: {
-        //     limit: Number.parseInt(response.headers['x-ratelimit-limit']),
-        //     remaining: Number.parseInt(response.headers['x-ratelimit-remaining']),
-        //     reset: parseDate(Number.parseInt(response.headers['x-ratelimit-reset']) * 1000),
-        //     resoure: response.headers['x-ratelimit-resource'],
-        //     used: Number.parseInt(response.headers['x-ratelimit-used']),
-        // },
+        rateLimit: {
+            limit: Number.parseInt(response.headers.get('x-ratelimit-limit')),
+            remaining: Number.parseInt(response.headers.get('x-ratelimit-remaining')),
+            reset: parseDate(Number.parseInt(response.headers.get('x-ratelimit-reset')) * 1000),
+            resoure: response.headers.get('x-ratelimit-resource'),
+            used: Number.parseInt(response.headers.get('x-ratelimit-used')),
+        },
     });
     return ret;
 }

package/lib/routes/github/issue.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import got from '@/utils/got';
 import { config } from '@/config';
 import MarkdownIt from 'markdown-it';
@@ -11,9 +11,32 @@ import { parseDate } from '@/utils/parse-date';
 
 export const route: Route = {
     path: '/issue/:user/:repo/:state?/:labels?',
-    categories: ['programming'],
-    example: '/github/issue/vuejs/core/all/wontfix',
-    parameters: { user: 'GitHub username', repo: 'GitHub repo name', state: 'the state of the issues. Can be either `open`, `closed`, or `all`. Default: `open`.', labels: 'a list of comma separated label names' },
+    categories: ['programming', 'popular'],
+    view: ViewType.Notifications,
+    example: '/github/issue/DIYgod/RSSHub/open',
+    parameters: {
+        user: 'GitHub username',
+        repo: 'GitHub repo name',
+        state: {
+            description: 'the state of the issues.',
+            default: 'open',
+            options: [
+                {
+                    label: 'Open',
+                    value: 'open',
+                },
+                {
+                    label: 'Closed',
+                    value: 'closed',
+                },
+                {
+                    label: 'All',
+                    value: 'all',
+                },
+            ],
+        },
+        labels: 'a list of comma separated label names',
+    },
     radar: [
         {
             source: ['github.com/:user/:repo/issues', 'github.com/:user/:repo/issues/:id', 'github.com/:user/:repo'],

package/lib/routes/github/notifications.ts

@@ -1,5 +1,5 @@
 import { Route } from '@/types';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { parseDate } from '@/utils/parse-date';
 
 const apiUrl = 'https://api.github.com';
@@ -10,7 +10,6 @@ export const route: Route = {
     path: '/notifications',
     categories: ['programming'],
     example: '/github/notifications',
-    parameters: {},
     features: {
         requireConfig: [
             {
@@ -18,11 +17,6 @@ export const route: Route = {
                 description: '',
             },
         ],
-        requirePuppeteer: false,
-        antiCrawler: false,
-        supportBT: false,
-        supportPodcast: false,
-        supportScihub: false,
     },
     radar: [
         {
@@ -45,10 +39,10 @@ async function handler(ctx) {
         'X-GitHub-Api-Version': '2022-11-28',
     };
 
-    const response = await got(`${apiUrl}/notifications`, {
+    const response = await ofetch.raw(`${apiUrl}/notifications`, {
         headers,
     });
-    const notifications = response.data;
+    const notifications = response._data;
 
     const items = notifications.map((item) => {
         let originUrl = item.subject.url ? item.subject.url.replace('https://api.github.com/repos/', 'https://github.com/') : 'https://github.com/notifications';
@@ -68,11 +62,11 @@ async function handler(ctx) {
         title: 'Github Notifications',
         item: notifications,
         rateLimit: {
-            limit: Number.parseInt(response.headers['X-RateLimit-Limit']),
-            remaining: Number.parseInt(response.headers['X-RateLimit-Remaining']),
-            reset: parseDate(Number.parseInt(response.headers['X-RateLimit-Reset']) * 1000),
-            resoure: response.headers['X-RateLimit-Resource'],
-            used: Number.parseInt(response.headers['X-RateLimit-Used']),
+            limit: Number.parseInt(response.headers['x-ratelimit-limit']),
+            remaining: Number.parseInt(response.headers['x-ratelimit-remaining']),
+            reset: parseDate(Number.parseInt(response.headers['x-ratelimit-reset']), 'X'),
+            resoure: response.headers['x-ratelimit-resource'],
+            used: Number.parseInt(response.headers['x-ratelimit-used']),
         },
     });
 

package/lib/routes/github/pulls.ts

@@ -1,5 +1,5 @@
 import { Route } from '@/types';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { config } from '@/config';
 import MarkdownIt from 'markdown-it';
 const md = MarkdownIt({
@@ -28,7 +28,7 @@ export const route: Route = {
         },
     ],
     name: 'Repo Pull Requests',
-    maintainers: [],
+    maintainers: ['hashman', 'TonyRL'],
     handler,
 };
 
@@ -45,8 +45,8 @@ async function handler(ctx) {
     if (config.github && config.github.access_token) {
         headers.Authorization = `token ${config.github.access_token}`;
     }
-    const response = await got(url, {
-        searchParams: {
+    const response = await ofetch(url, {
+        query: {
             state,
             labels,
             sort: 'created',
@@ -55,7 +55,7 @@ async function handler(ctx) {
         },
         headers,
     });
-    const data = response.data.filter((item) => item.pull_request);
+    const data = response.filter((item) => item.pull_request);
 
     return {
         allowEmpty: true,

package/lib/routes/github/search.ts

@@ -1,6 +1,5 @@
-import { Route } from '@/types';
-import got from '@/utils/got';
-import { load } from 'cheerio';
+import { DataItem, Route } from '@/types';
+import ofetch from '@/utils/ofetch';
 import * as url from 'node:url';
 
 const host = 'https://github.com';
@@ -40,22 +39,25 @@ async function handler(ctx) {
 
     const suffix = 'search?o='.concat(order, '&q=', encodeURIComponent(query), '&s=', sort, '&type=Repositories');
     const link = url.resolve(host, suffix);
-    const response = await got.get(link);
-    const $ = load(response.data);
+    const response = await ofetch(link, {
+        headers: {
+            accept: 'application/json',
+        },
+    });
 
-    const out = $('.repo-list li')
-        .slice(0, 10)
-        .map(function () {
-            const a = $(this).find('.f4.text-normal > a');
-            const single = {
-                title: a.text(),
-                author: a.text().split('/')[0].trim(),
-                link: host.concat(a.attr('href')),
-                description: $(this).find('div p').text().trim(),
-            };
-            return single;
-        })
-        .get();
+    const out = response.payload.results.map((item) => {
+        const {
+            repo: { repository },
+            hl_trunc_description,
+        } = item;
+
+        return {
+            title: repository.name,
+            author: repository.owner_login,
+            link: host.concat(`/${repository.owner_login}/${repository.name}`),
+            description: hl_trunc_description,
+        } as DataItem;
+    });
 
     return {
         allowEmpty: true,

package/lib/routes/github/star.ts

@@ -1,12 +1,13 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import got from '@/utils/got';
 import { config } from '@/config';
 import ConfigNotFoundError from '@/errors/types/config-not-found';
 
 export const route: Route = {
     path: '/stars/:user/:repo',
-    categories: ['programming'],
-    example: '/github/stars/DIYGod/RSSHub',
+    categories: ['programming', 'popular'],
+    example: '/github/stars/DIYgod/RSSHub',
+    view: ViewType.Notifications,
     parameters: { user: 'GitHub username', repo: 'GitHub repo name' },
     features: {
         requireConfig: [

package/lib/routes/github/trending.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import { getCurrentPath } from '@/utils/helpers';
 const __dirname = getCurrentPath(import.meta.url);
 
@@ -11,12 +11,34 @@ import ConfigNotFoundError from '@/errors/types/config-not-found';
 
 export const route: Route = {
     path: '/trending/:since/:language/:spoken_language?',
-    categories: ['programming'],
+    categories: ['programming', 'popular'],
     example: '/github/trending/daily/javascript/en',
+    view: ViewType.Articles,
     parameters: {
-        since: "time frame, available in [Trending page](https://github.com/trending/javascript?since=monthly) 's URL, possible values are: `daily`, `weekly` or `monthly`",
-        language: "the feed language, available in [Trending page](https://github.com/trending/javascript?since=monthly) 's URL, don't filter option is `any`",
-        spoken_language: "natural language, available in [Trending page](https://github.com/trending/javascript?since=monthly) 's URL",
+        since: {
+            description: 'time range',
+            options: [
+                {
+                    value: 'daily',
+                    label: 'Today',
+                },
+                {
+                    value: 'weekly',
+                    label: 'This week',
+                },
+                {
+                    value: 'monthly',
+                    label: 'This month',
+                },
+            ],
+        },
+        language: {
+            description: "the feed language, available in [Trending page](https://github.com/trending/javascript?since=monthly) 's URL, don't filter option is `any`",
+            default: 'any',
+        },
+        spoken_language: {
+            description: "natural language, available in [Trending page](https://github.com/trending/javascript?since=monthly) 's URL",
+        },
     },
     features: {
         requireConfig: [

package/lib/routes/github/wiki.ts

@@ -32,7 +32,15 @@ export const route: Route = {
 async function handler(ctx) {
     const { user, repo, page } = ctx.req.param();
 
-    const url = `${baseUrl}/${user}/${repo}/wiki${page ? `/${page}` : ''}/_history`;
+    let url = `${baseUrl}/${user}/${repo}/wiki${page ? `/${page}` : ''}/_history`;
+
+    // Fetch page slug. History fetched with no page specified has no <a> tag for commit.
+    if (!page) {
+        const { data } = await got(`${baseUrl}/${user}/${repo}/wiki`);
+        const $ = load(data);
+
+        url = `${baseUrl}${$('a[href$=_history]').attr('href')}`;
+    }
 
     const { data } = await got(url);
     const $ = load(data);

package/lib/routes/go/mhlw/pdf.ts

@@ -0,0 +1,87 @@
+import { Route } from '@/types';
+
+import got from '@/utils/got';
+import { load } from 'cheerio';
+
+export const handler = async (ctx) => {
+    const { category = 'stf/seisakunitsuite/bunya/houkokusuunosuii' } = ctx.req.param();
+    const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 30;
+
+    const rootUrl = 'https://www.mhlw.go.jp';
+    const currentUrl = new URL(category.endsWith('.html') ? category : `${category}.html`, rootUrl).href;
+
+    const { data: response } = await got(currentUrl);
+
+    const $ = load(response);
+
+    const language = $('html').prop('lang');
+
+    const items = $('a[data-icon="pdf"]')
+        .slice(0, limit)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const title = item.find('font').text() || item.text();
+            const link = new URL(item.prop('href'), rootUrl).href;
+
+            return {
+                title,
+                link,
+                language,
+                enclosure_url: link,
+                enclosure_type: link ? 'application/pdf' : undefined,
+                enclosure_title: title,
+            };
+        });
+
+    const image = new URL($('div.m-headerLogo img').first().prop('src'), rootUrl).href;
+
+    return {
+        title: $('title').text(),
+        description: $('meta[property="og:description"]').prop('content'),
+        link: currentUrl,
+        item: items,
+        allowEmpty: true,
+        image,
+        author: $('meta[property="og:site_name"]').prop('content'),
+        language,
+    };
+};
+
+export const route: Route = {
+    path: '/mhlw/pdf/:category{.+}?',
+    name: 'PDF',
+    url: 'www.mhlw.go.jp',
+    maintainers: ['nczitzk'],
+    handler,
+    example: '/go/mhlw/pdf/stf/seisakunitsuite/bunya/houkokusuunosuii',
+    parameters: { category: 'Category, `stf/seisakunitsuite/bunya/houkokusuunosuii` as 新型コロナウイルス感染症の定点当たり報告数の推移 by default' },
+    description: `:::tip
+  Subscribing to this route will give you access to all PDF files on this page.
+
+  If you subscribe to [新型コロナウイルス感染症の定点当たり報告数の推移](https://www.mhlw.go.jp/stf/seisakunitsuite/bunya/houkokusuunosuii.html)，where the URL is \`https://www.mhlw.go.jp/stf/seisakunitsuite/bunya/houkokusuunosuii.html\`, extract the part \`https://www.mhlw.go.jp/\` to the end, which is \`.html\`, and use it as the parameter to fill in. Therefore, the route will be [\`/go/mhlw/stf/seisakunitsuite/bunya/houkokusuunosuii\`](https://rsshub.app/go/mhlw/stf/seisakunitsuite/bunya/houkokusuunosuii).
+  :::
+  `,
+    categories: ['government'],
+
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportRadar: true,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['www.mhlw.go.jp'],
+            target: (_, url) => {
+                const category = new URL(url).href.match(/mhlw\.go\.jp\/(.*)$/)?.[1] ?? undefined;
+
+                return `/mhlw/pdf${category ? `/${category}` : ''}`;
+            },
+        },
+    ],
+};

package/lib/routes/go/namespace.ts

@@ -0,0 +1,8 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'JapanGov',
+    url: 'go.jp',
+    categories: ['government'],
+    description: 'The Government of Japan',
+};