rsshub 1.0.0-master.f71451d → 1.0.0-master.f72af1b

package/lib/routes/devolverdigital/blog.ts

@@ -1,11 +1,10 @@
-import { Route } from '@/types';
-import cache from '@/utils/cache';
-import got from '@/utils/got';
+import { DataItem, Route } from '@/types';
 import { load } from 'cheerio';
+import ofetch from '@/utils/ofetch';
 
 export const route: Route = {
     path: '/blog',
-    categories: ['blog'],
+    categories: ['game'],
     example: '/devolverdigital/blog',
     parameters: {},
     features: {
@@ -28,44 +27,7 @@ export const route: Route = {
 };
 
 async function handler() {
-    const baseUrl = 'https://www.devolverdigital.com/blog';
-    const { data: response } = await got(baseUrl);
-
-    const $ = load(response);
-    const nextData = JSON.parse($('#__NEXT_DATA__').text());
-
-    const items = await Promise.all(
-        nextData.props.pageProps.posts.map((postData) => {
-            const postUrl = `${baseUrl}/post/${postData.id}`;
-            return cache.tryGet(postUrl, async () => {
-                const { data: postPage } = await got(postUrl);
-
-                const $page = load(postPage);
-                $page('noscript').remove();
-                const postContent = $page('div.flex > div > div > div > div:not([class])');
-
-                // img resource redirection and
-                // clean up absolute layouts for img and span
-                const imageUrls = postData.body.filter((item) => item.type === 'upload' && item.value.cloudinary.resource_type === 'image').map((item) => item.value.cloudinary.secure_url);
-                const allImageSpans = postContent.find('span > img').parent();
-                allImageSpans.each((spanIndex, span) => {
-                    $(span).attr('style', $(span).attr('style').replace('position:absolute', ''));
-                    const img = $(span).find('img');
-                    img.attr('src', imageUrls[spanIndex]);
-                    img.attr('style', img.attr('style').replace('position:absolute', '').replace('width:0', '').replace('height:0', ''));
-                });
-
-                return {
-                    title: postData.title,
-                    link: postUrl,
-                    author: postData.author,
-                    pubDate: Date.parse(postData.createdAt),
-                    updated: Date.parse(postData.updatedAt),
-                    description: postContent.html(),
-                };
-            });
-        })
-    );
+    const items = await fetchPage();
 
     return {
         title: 'DevolverDigital Blog',
@@ -74,3 +36,58 @@ async function handler() {
         item: items,
     };
 }
+
+async function fetchPage() {
+    const baseUrl = 'https://www.devolverdigital.com/blog';
+    const response = await ofetch(baseUrl);
+    const $ = load(response, { scriptingEnabled: false });
+
+    // Extract all posts of this page
+    const $titleDivs = $('div.w-full.flex.justify-center.py-4.bg-red-400.undefined');
+    const $contentDivs = $('div.bg-gray-800.flex.justify-center.font-sm.py-4');
+    const items: DataItem[] = $titleDivs.toArray().map((titleDiv, index) => {
+        const content = $contentDivs[index];
+        const postAuthor = parsePostAuthor($, titleDiv);
+        const postDate = parsePostDate($, titleDiv);
+        const postTitle = $(titleDiv).find('h1').text();
+        const postLink = $(content).find('div.ml-auto.flex.items-center a').attr('href');
+        // Modify the src attribute of the image
+        parsePostImages($, content);
+        const postContent = $.html($(content).find('div.cms-content'));
+        return {
+            title: postTitle,
+            link: postLink,
+            author: postAuthor,
+            pubDate: postDate,
+            description: postContent,
+        };
+    });
+
+    return items;
+}
+
+function parsePostAuthor($, titleDiv) {
+    const $postAuthorElement = $(titleDiv).find('div.font-xs.leading-none.mb-1');
+    return $postAuthorElement.text().replace('By ', '') || 'Devolver Digital';
+}
+
+function parsePostDate($, titleDiv) {
+    const dateStr = $(titleDiv).find('div.font-2xs.leading-none.mb-1').text();
+    const cleanedDateStr = dateStr.replace(/(\d+)(st|nd|rd|th)/, '$1');
+    return new Date(cleanedDateStr);
+}
+
+function parsePostImages($, content) {
+    $(content)
+        .find('img')
+        .each((index, img) => {
+            const $img = $(img);
+            const src = $img.attr('src') || '';
+            if (src.startsWith('/_next/image')) {
+                const srcSet = $img.attr('srcset') || '';
+                const actualSrc = srcSet.split(',').pop()?.split(' ')[0] || src;
+                $img.attr('src', actualSrc);
+            }
+            $img.removeAttr('loading').removeAttr('decoding').removeAttr('data-nimg').removeAttr('style').removeAttr('sizes').removeAttr('srcset').removeAttr('referrerpolicy');
+        });
+}

package/lib/routes/devtrium/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Devtrium',
+    url: 'devtrium.com',
+};

package/lib/routes/devtrium/posts.ts

@@ -0,0 +1,55 @@
+import { DataItem, Route } from '@/types';
+import { load } from 'cheerio';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+
+export const route: Route = {
+    path: '/',
+    categories: ['programming'],
+    example: '/devtrium',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['devtrium.com'],
+        },
+    ],
+    name: 'Official Blogs',
+    maintainers: ['Xy2002'],
+    handler,
+    url: 'devtrium.com',
+};
+
+async function handler() {
+    const items = await fetchPage();
+
+    return {
+        title: 'Devtrium',
+        language: 'en-us',
+        item: items,
+        link: 'https://devtrium.com',
+    };
+}
+
+async function fetchPage() {
+    const baseUrl = 'https://devtrium.com';
+    const response = await ofetch(baseUrl);
+    const $ = load(response, { scriptingEnabled: false });
+
+    // Extract all posts of this page
+    const data = JSON.parse($('script#__NEXT_DATA__').text());
+    const items: DataItem[] = data.props.pageProps.posts.map((post) => ({
+        title: post.title,
+        link: `${baseUrl}/posts/${post.slug}`,
+        description: post.description,
+        pubDate: parseDate(post.date),
+    }));
+    return items;
+}

package/lib/routes/dewu/declaration.ts

@@ -0,0 +1,86 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
+
+const typeMap = {
+    '1010580020': '技术变更',
+    '1014821004': '服务市场规则中心',
+    '1011202692': '规则变更',
+    '1010568195': '维护公告',
+};
+
+/**
+ *
+ * @param ctx {import('koa').Context}
+ */
+export const route: Route = {
+    path: '/declaration/:categoryId?',
+    categories: ['programming'],
+    example: '/dewu/declaration/1010580020',
+    parameters: { categoryId: '公告分类, 可在页面URL获取 默认为1010580020' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: '平台公告',
+    maintainers: ['blade0910'],
+    handler,
+    description: `| 类型             | type       |
+  | ---------------- | ---------- |
+  | 技术变更         | 1010580020 |
+  | 服务市场规则中心 | 1014821004 |
+  | 规则变更         | 1011202692 |
+  | 维护公告         | 1010568195 |`,
+};
+
+async function handler(ctx) {
+    const categoryId = ctx.req.param('categoryId') || '1010580020';
+    const response = await got({
+        method: 'post',
+        url: 'https://open.dewu.com/api/v1/h5/merchant-study/open/document/listDocument',
+        headers: {
+            'Content-Type': 'application/json',
+        },
+        json: {
+            categoryId,
+        },
+    });
+
+    const list = response.data.data.map((item) => ({
+        title: item.title,
+        id: item.id,
+        link: `https://open.dewu.com/#/declaration/read?articleId=${item.id}`,
+        pubDate: timezone(parseDate(item.publishTime), +8),
+    }));
+
+    const result = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const itemResponse = await got({
+                    method: 'post',
+                    url: 'https://open.dewu.com/api/v1/h5/merchant-study/open/document/getDocumentDetail',
+                    headers: {
+                        'Content-Type': 'application/json',
+                    },
+                    json: {
+                        documentId: item.id,
+                    },
+                });
+                item.description = itemResponse.data.data.content;
+                return item;
+            })
+        )
+    );
+
+    return {
+        title: `得物开放平台 - ${typeMap[categoryId] ?? '平台公告'}`,
+        link: 'https://open.dewu.com/#/declaration/read',
+        item: result,
+    };
+}

package/lib/routes/dewu/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '得物',
+    url: 'dewu.com',
+};

package/lib/routes/dhu/jiaowu/news.ts

@@ -2,13 +2,15 @@ import { Route } from '@/types';
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
-import timezone from '@/utils/timezone';
+import cache from '@/utils/cache';
 
-const base_url = 'https://jw.dhu.edu.cn';
+const baseUrl = 'https://jw.dhu.edu.cn';
 
 const map = {
     student: '/tzggwxszl/list.htm',
     teacher: '/tzggwjszl/list.htm',
+    class: '/tzggwxkzl_19850/list.htm',
+    fxzy: '/fxzy/list.htm',
 };
 export const route: Route = {
     path: '/jiaowu/news/:type?',
@@ -26,32 +28,59 @@ export const route: Route = {
     name: '教务处通知',
     maintainers: ['KiraKiseki'],
     handler,
-    description: `| 学生专栏 | 教师专栏 |
-  | -------- | -------- |
-  | student  | teacher  |`,
+    description: `| 学生专栏 | 教师专栏 | 选课专栏（仅选课期间开放） | 辅修专业 |
+  | -------- | -------- | -------- | -------- |
+  | student  | teacher  | class    | fxzy     |`,
 };
 
 async function handler(ctx) {
-    const type = ctx.req.param('type');
-    const link = Object.hasOwn(map, type) ? `${base_url}${map[type]}` : `${base_url}/tzggwxszl/list.htm`;
-    const response = await got({
-        method: 'get',
-        url: link,
-        headers: {
-            Referer: base_url,
-        },
-    });
-
-    const $ = load(response.data);
+    const type = ctx.req.param('type') || 'student';
+    const link = `${baseUrl}${map[type]}`;
+    const { data: response } = await got(link);
+
+    const $ = load(response);
+
+    const items = await Promise.all(
+        $('.list2 > li')
+            .toArray()
+            .map(async (item) => {
+                item = $(item);
+                const newsTitle = item.find('.news_title > a');
+                const newsMeta = item.find('.news_meta');
+
+                // article meta
+                const link = newsTitle.attr('href');
+                const title = newsTitle.text();
+                const pubDate = parseDate(newsMeta.text(), 'YYYY-MM-DD', 'zh-cn');
+
+                // fetch article content and return item using cache.tryGet
+                // url as cache key
+                const url = `${baseUrl}${link}`;
+                return await cache.tryGet(url, async () => {
+                    // fetch article content
+                    // some contents are only available for internal network
+                    let description = '';
+                    try {
+                        const { data: response } = await got(url);
+                        const $ = load(response);
+                        description = $('.wp_articlecontent').first().html() ?? '';
+                    } catch {
+                        description = '';
+                    }
+
+                    return {
+                        title,
+                        link,
+                        pubDate,
+                        description,
+                    };
+                });
+            })
+    );
+
     return {
-        link: base_url,
         title: '东华大学教务处-' + $('.col_title').text(),
-        item: $('.list_item')
-            .map((_, elem) => ({
-                link: new URL($('a', elem).attr('href'), base_url),
-                title: $('a', elem).attr('title'),
-                pubDate: timezone(parseDate($('.Article_PublishDate', elem).text()), +8),
-            }))
-            .get(),
+        link,
+        item: items,
     };
 }

package/lib/routes/dhu/namespace.ts

@@ -2,5 +2,5 @@ import type { Namespace } from '@/types';
 
 export const namespace: Namespace = {
     name: '东华大学',
-    url: 'gs.dhu.edu.cn',
+    url: 'www.dhu.edu.cn',
 };

package/lib/routes/dhu/xxgk/news.ts

@@ -2,9 +2,10 @@ import { Route } from '@/types';
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
-import timezone from '@/utils/timezone';
+import cache from '@/utils/cache';
 
-const base_url = 'https://xxgk.dhu.edu.cn/1737/list.htm';
+const siteUrl = 'https://xxgk.dhu.edu.cn';
+const baseUrl = 'https://xxgk.dhu.edu.cn/1737/list.htm';
 
 export const route: Route = {
     path: '/xxgk/news',
@@ -25,25 +26,52 @@ export const route: Route = {
 };
 
 async function handler() {
-    const link = base_url;
-    const response = await got({
-        method: 'get',
-        url: link,
-        headers: {
-            Referer: base_url,
-        },
-    });
-
-    const $ = load(response.data);
+    const link = baseUrl;
+    const { data: response } = await got(link);
+
+    const $ = load(response);
+
+    const items = await Promise.all(
+        $('.cols_list > li')
+            .toArray()
+            .map(async (item) => {
+                item = $(item);
+                const colsTitle = item.find('.cols_title > a');
+                const colsMeta = item.find('.cols_meta');
+
+                // article meta
+                const link = colsTitle.attr('href');
+                const title = colsTitle.text();
+                const pubDate = parseDate(colsMeta.text(), 'YYYY-MM-DD', 'zh-cn');
+
+                // fetch article content and return item using cache.tryGet
+                // url as cache key
+                const url = `${siteUrl}${link}`;
+                return await cache.tryGet(url, async () => {
+                    // fetch article content
+                    // some contents are only available for internal network
+                    let description = '';
+                    try {
+                        const { data: response } = await got(url);
+                        const $ = load(response);
+                        description = $('.wp_articlecontent').first().html() ?? '';
+                    } catch {
+                        description = '';
+                    }
+
+                    return {
+                        title,
+                        link,
+                        pubDate,
+                        description,
+                    };
+                });
+            })
+    );
+
     return {
-        link: base_url,
         title: '东华大学信息公开网-最新公开信息',
-        item: $('.cols')
-            .map((_, elem) => ({
-                link: new URL($('a', elem).attr('href'), base_url),
-                title: $('a', elem).attr('title'),
-                pubDate: timezone(parseDate($('.cols_meta', elem).text()), +8),
-            }))
-            .get(),
+        link,
+        item: items,
     };
 }

package/lib/routes/digitalcameraworld/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Digital Camera World',
+    url: 'digitalcameraworld.com',
+};

package/lib/routes/digitalcameraworld/news.ts

@@ -0,0 +1,61 @@
+import { Route } from '@/types';
+import { load } from 'cheerio';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+const host = 'https://www.digitalcameraworld.com';
+export const route: Route = {
+    path: '/news',
+    categories: ['new-media'],
+    example: '/digitalcameraworld/news',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['digitalcameraworld.com/'],
+        },
+    ],
+    name: 'News',
+    maintainers: ['EthanWng97'],
+    handler,
+};
+
+async function handler() {
+    const rssUrl = `${host}/feeds.xml`;
+
+    const response = await ofetch(rssUrl);
+
+    const $ = load(response, {
+        xmlMode: true,
+    });
+
+    const items = $('item')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            let description = item.find(String.raw`dc\:content`).text();
+            description = $('<div>').html(description);
+            description.find('.vanilla-image-block').removeAttr('style');
+            description.find('.fancy-box').remove();
+
+            return {
+                title: item.find('title').text(),
+                pubDate: parseDate(item.find('pubDate').text()),
+                link: item.find('link').text(),
+                description: description.html(),
+            };
+        });
+
+    return {
+        title: 'Digital Camera World',
+        link: host,
+        description: 'Camera news, reviews and features',
+        item: items,
+    };
+}

package/lib/routes/discord/channel.ts

@@ -1,8 +1,7 @@
-import { Route } from '@/types';
+import { DataItem, Route } from '@/types';
 import { getCurrentPath } from '@/utils/helpers';
 const __dirname = getCurrentPath(import.meta.url);
 
-import cache from '@/utils/cache';
 import { config } from '@/config';
 import { parseDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
@@ -19,7 +18,7 @@ export const route: Route = {
         requireConfig: [
             {
                 name: 'DISCORD_AUTHORIZATION',
-                description: '',
+                description: 'Discord authorization header from the browser',
             },
         ],
         requirePuppeteer: false,
@@ -45,17 +44,17 @@ async function handler(ctx) {
     const { authorization } = config.discord;
     const channelId = ctx.req.param('channelId');
 
-    const channelInfo = await getChannel(channelId, authorization, cache.tryGet);
-    const messagesRaw = await getChannelMessages(channelId, authorization, cache.tryGet, ctx.req.query('limit') ?? 100);
+    const channelInfo = await getChannel(channelId, authorization);
+    const messagesRaw = await getChannelMessages(channelId, authorization, ctx.req.query('limit') ?? 100);
     const { name: channelName, topic: channelTopic, guild_id: guildId } = channelInfo;
 
-    const guildInfo = await getGuild(guildId, authorization, cache.tryGet);
+    const guildInfo = await getGuild(guildId, authorization);
     const { name: guildName, icon: guidIcon } = guildInfo;
 
     const messages = messagesRaw.map((message) => ({
-        title: message.content,
-        description: art(path.join(__dirname, 'templates/message.art'), { message }),
-        author: `${message.author.username}#${message.author.discriminator}`,
+        title: message.content.split('\n')[0],
+        description: art(path.join(__dirname, 'templates/message.art'), { message, guildInfo }),
+        author: `${message.author.global_name ?? message.author.username}(${message.author.username})`,
         pubDate: parseDate(message.timestamp),
         updated: message.edited_timestamp ? parseDate(message.edited_timestamp) : undefined,
         category: `#${channelName}`,
@@ -67,6 +66,7 @@ async function handler(ctx) {
         description: channelTopic,
         link: `${baseUrl}/channels/${guildId}/${channelId}`,
         image: `https://cdn.discordapp.com/icons/${guildId}/${guidIcon}.webp`,
-        item: messages,
+        item: messages as unknown as DataItem[],
+        allowEmpty: true,
     };
 }

package/lib/routes/discord/discord-api.ts

@@ -1,55 +1,50 @@
-import got from '@/utils/got';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
 import { config } from '@/config';
+import { RESTGetAPIGuildResult, RESTGetAPIGuildChannelsResult, RESTGetAPIChannelResult, RESTGetAPIChannelMessagesQuery, RESTGetAPIChannelMessagesResult } from 'discord-api-types/rest/v10';
 
-const baseUrl = 'https://discord.com';
+export const baseUrl = 'https://discord.com';
 const apiUrl = `${baseUrl}/api/v10`;
 
-const getGuild = (guildId, authorization, tryGet) =>
-    tryGet(`discord:guilds:${guildId}`, async () => {
-        const response = await got(`${apiUrl}/guilds/${guildId}`, {
+export const getGuild = (guildId, authorization) =>
+    cache.tryGet(`discord:guilds:${guildId}`, () =>
+        ofetch(`${apiUrl}/guilds/${guildId}`, {
             headers: {
                 authorization,
             },
-        });
-        return response.data;
-    });
+        })
+    ) as Promise<RESTGetAPIGuildResult>;
 
-const getGuildChannels = (guildId, authorization, tryGet) =>
-    tryGet(`discord:guilds:${guildId}:channels`, async () => {
-        const response = await got(`${apiUrl}/guilds/${guildId}/channels`, {
+export const getGuildChannels = (guildId, authorization) =>
+    cache.tryGet(`discord:guilds:${guildId}:channels`, () =>
+        ofetch(`${apiUrl}/guilds/${guildId}/channels`, {
             headers: {
                 authorization,
             },
-        });
-        return response.data;
-    });
+        })
+    ) as Promise<RESTGetAPIGuildChannelsResult>;
 
-const getChannel = (channelId, authorization, tryGet) =>
-    tryGet(`discord:channels:${channelId}`, async () => {
-        const response = await got(`${apiUrl}/channels/${channelId}`, {
+export const getChannel = (channelId, authorization) =>
+    cache.tryGet(`discord:channels:${channelId}`, () =>
+        ofetch(`${apiUrl}/channels/${channelId}`, {
             headers: {
                 authorization,
             },
-        });
-        return response.data;
-    });
+        })
+    ) as Promise<RESTGetAPIChannelResult>;
 
-const getChannelMessages = (channelId, authorization, tryGet, limit = 100) =>
-    tryGet(
+export const getChannelMessages = (channelId, authorization, limit = 100) =>
+    cache.tryGet(
         `discord:channels:${channelId}:messages`,
-        async () => {
-            const response = await got(`${apiUrl}/channels/${channelId}/messages`, {
+        () =>
+            ofetch(`${apiUrl}/channels/${channelId}/messages`, {
                 headers: {
                     authorization,
                 },
-                searchParams: {
+                query: {
                     limit,
-                },
-            });
-            return response.data;
-        },
+                } as RESTGetAPIChannelMessagesQuery,
+            }),
         config.cache.routeExpire,
         false
-    );
-
-export { baseUrl, getGuild, getGuildChannels, getChannel, getChannelMessages };
+    ) as Promise<RESTGetAPIChannelMessagesResult>;