rsshub 1.0.0-master.ff487f3 → 1.0.0-master.ff76aa7

package/lib/routes/rss3/index.ts

@@ -1,6 +1,8 @@
-import { Route } from '@/types';
+import { Route, type DataItem } from '@/types';
 
+import { camelcaseKeys } from '@/utils/camelcase-keys';
 import ofetch from '@/utils/ofetch';
+import { renderItemActionToHTML } from '@rss3/sdk';
 
 export const route: Route = {
     path: '/:account/:network?/:tag?',
@@ -132,17 +134,24 @@ async function handler(ctx) {
     return {
         title: `${account} activities`,
         link: 'https://rss3.io',
-        item: data.map((item) => ({
-            title: `New ${item.tag} ${item.type} action on ${item.network}`,
-            description: `New ${item.tag} ${item.type} action on ${item.network}<br /><br />From: ${item.from}<br/>To: ${item.to}`,
-            link: item.actions?.[0]?.related_urls?.[0],
-            guid: item.id,
-            author: [
-                {
-                    name: item.owner,
-                    avatar: `https://cdn.stamp.fyi/avatar/eth:${item.owner}`,
-                },
-            ],
-        })),
+        item: data.map((item) => {
+            const content = renderItemActionToHTML(camelcaseKeys(item.actions));
+
+            const description = `New ${item.tag} ${item.type} action on ${item.network}<br /><br />From: ${item.from}<br/>To: ${item.to}`;
+            return {
+                title: `New ${item.tag} ${item.type} action on ${item.network}`,
+                description: content ?? description,
+                link: item.actions?.[0]?.related_urls?.[0],
+                guid: item.id,
+                author: [
+                    {
+                        name: item.owner,
+                        avatar: `https://cdn.stamp.fyi/avatar/eth:${item.owner}`,
+                    },
+                ],
+
+                _extra: { raw: item },
+            } as DataItem;
+        }),
     };
 }

package/lib/routes/rss3/interfaces/metadata.ts

@@ -0,0 +1,67 @@
+import {
+    CollectibleApproval,
+    CollectibleBurn,
+    CollectibleMint,
+    CollectibleTrade,
+    CollectibleTransfer,
+    ExchangeLiquidity,
+    ExchangeStaking,
+    ExchangeSwap,
+    MetaverseBurn,
+    MetaverseMint,
+    MetaverseTrade,
+    MetaverseTransfer,
+    SocialComment,
+    SocialDelete,
+    SocialMint,
+    SocialPost,
+    SocialProfile,
+    SocialProxy,
+    SocialRevise,
+    SocialReward,
+    SocialShare,
+    StakeStaking,
+    StakeTransaction,
+    StakerProfitSnapshot,
+    TransactionApproval,
+    TransactionBridge,
+    TransactionBurn,
+    TransactionEvent,
+    TransactionMint,
+    TransactionTransfer,
+} from '@rss3/sdk';
+export type RSS3DataModels = {
+    CollectibleApproval: CollectibleApproval;
+    CollectibleBurn: CollectibleBurn;
+    CollectibleMint: CollectibleMint;
+    CollectibleTrade: CollectibleTrade;
+    CollectibleTransfer: CollectibleTransfer;
+    MetaverseBurn: MetaverseBurn;
+    MetaverseMint: MetaverseMint;
+    MetaverseTrade: MetaverseTrade;
+    MetaverseTransfer: MetaverseTransfer;
+    SocialComment: SocialComment;
+    SocialDelete: SocialDelete;
+    SocialMint: SocialMint;
+    SocialPost: SocialPost;
+    SocialProfile: SocialProfile;
+    SocialProxy: SocialProxy;
+    SocialRevise: SocialRevise;
+    SocialReward: SocialReward;
+    SocialShare: SocialShare;
+    StakeStaking: StakeStaking;
+    StakeTransaction: StakeTransaction;
+    StakerProfitSnapshot: StakerProfitSnapshot;
+    TransactionApproval: TransactionApproval;
+    TransactionBridge: TransactionBridge;
+    TransactionBurn: TransactionBurn;
+    TransactionEvent: TransactionEvent;
+    TransactionMint: TransactionMint;
+    TransactionTransfer: TransactionTransfer;
+    ExchangeLiquidity: ExchangeLiquidity;
+    ExchangeStaking: ExchangeStaking;
+    ExchangeSwap: ExchangeSwap;
+};
+export type GetRSS3DataMetadata<FirstKey extends string, SecondKey extends string> = `${Capitalize<FirstKey>}${Capitalize<SecondKey>}` extends keyof RSS3DataModels
+    ? RSS3DataModels[`${Capitalize<FirstKey>}${Capitalize<SecondKey>}`]
+    : null;

package/lib/routes/rsshub/transform/html.ts

@@ -1,8 +1,10 @@
-import { Route } from '@/types';
+import { DataItem, Route } from '@/types';
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import { config } from '@/config';
 import ConfigNotFoundError from '@/errors/types/config-not-found';
+import sanitizeHtml from 'sanitize-html';
+import cache from '@/utils/cache';
 
 export const route: Route = {
     path: '/transform/html/:url/:routeParams',
@@ -23,24 +25,25 @@ export const route: Route = {
         supportScihub: false,
     },
     name: 'Transformation - HTML',
-    maintainers: ['ttttmr'],
-    handler,
+    maintainers: ['ttttmr', 'hyoban'],
     description: `Pass URL and transformation rules to convert HTML/JSON into RSS.
 
 Specify options (in the format of query string) in parameter \`routeParams\` parameter to extract data from HTML.
 
-| Key               | Meaning                                                        | Accepted Values | Default                |
-| ----------------- | -------------------------------------------------------------- | --------------- | ---------------------- |
-| \`title\`           | The title of the RSS                                           | \`string\`        | Extract from \`<title>\` |
-| \`item\`            | The HTML elements as \`item\` using CSS selector                 | \`string\`        | html                   |
-| \`itemTitle\`       | The HTML elements as \`title\` in \`item\` using CSS selector      | \`string\`        | \`item\` element         |
-| \`itemTitleAttr\`   | The attributes of \`title\` element as title                     | \`string\`        | Element text           |
-| \`itemLink\`        | The HTML elements as \`link\` in \`item\` using CSS selector       | \`string\`        | \`item\` element         |
-| \`itemLinkAttr\`    | The attributes of \`link\` element as link                       | \`string\`        | \`href\`                 |
-| \`itemDesc\`        | The HTML elements as \`descrption\` in \`item\` using CSS selector | \`string\`        | \`item\` element         |
-| \`itemDescAttr\`    | The attributes of \`descrption\` element as description          | \`string\`        | Element html           |
-| \`itemPubDate\`     | The HTML elements as \`pubDate\` in \`item\` using CSS selector    | \`string\`        | \`item\` element         |
-| \`itemPubDateAttr\` | The attributes of \`pubDate\` element as pubDate                 | \`string\`        | Element html           |
+| Key                 | Meaning                                                                                                       | Accepted Values | Default                  |
+| ------------------- | ------------------------------------------------------------------------------------------------------------- | --------------- | ------------------------ |
+| \`title\`           | The title of the RSS                                                                                          | \`string\`      | Extract from \`<title>\` |
+| \`item\`            | The HTML elements as \`item\` using CSS selector                                                              | \`string\`      | html                     |
+| \`itemTitle\`       | The HTML elements as \`title\` in \`item\` using CSS selector                                                 | \`string\`      | \`item\` element         |
+| \`itemTitleAttr\`   | The attributes of \`title\` element as title                                                                  | \`string\`      | Element text             |
+| \`itemLink\`        | The HTML elements as \`link\` in \`item\` using CSS selector                                                  | \`string\`      | \`item\` element         |
+| \`itemLinkAttr\`    | The attributes of \`link\` element as link                                                                    | \`string\`      | \`href\`                 |
+| \`itemDesc\`        | The HTML elements as \`descrption\` in \`item\` using CSS selector                                            | \`string\`      | \`item\` element         |
+| \`itemDescAttr\`    | The attributes of \`descrption\` element as description                                                       | \`string\`      | Element html             |
+| \`itemPubDate\`     | The HTML elements as \`pubDate\` in \`item\` using CSS selector                                               | \`string\`      | \`item\` element         |
+| \`itemPubDateAttr\` | The attributes of \`pubDate\` element as pubDate                                                              | \`string\`      | Element html             |
+| \`itemContent\`     | The HTML elements as \`description\` in \`item\` using CSS selector ( in \`itemLink\` page for full content ) | \`string\`      |                          |
+| \`encoding\`        | The encoding of the HTML content                                                                              | \`string\`      | utf-8                    |
 
   Parameters parsing in the above example:
 
@@ -54,66 +57,102 @@ Specify options (in the format of query string) in parameter \`routeParams\` par
   | Parameter | Value                           |
   | --------- | ------------------------------- |
   | \`item\`    | \`div[class='post-content'] p a\` |`,
-};
+    handler: async (ctx) => {
+        if (!config.feature.allow_user_supply_unsafe_domain) {
+            throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
+        }
+        const url = ctx.req.param('url');
+        const response = await got({
+            method: 'get',
+            url,
+            responseType: 'arrayBuffer',
+        });
 
-async function handler(ctx) {
-    if (!config.feature.allow_user_supply_unsafe_domain) {
-        throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
-    }
-    const url = ctx.req.param('url');
-    const response = await got({
-        method: 'get',
-        url,
-    });
-
-    const routeParams = new URLSearchParams(ctx.req.param('routeParams'));
-    const $ = load(response.data);
-    const rssTitle = routeParams.get('title') || $('title').text();
-    const item = routeParams.get('item') || 'html';
-    const items = $(item)
-        .toArray()
-        .map((item) => {
-            try {
-                item = $(item);
-
-                const titleEle = routeParams.get('itemTitle') ? item.find(routeParams.get('itemTitle')) : item;
-                const title = routeParams.get('itemTitleAttr') ? titleEle.attr(routeParams.get('itemTitleAttr')) : titleEle.text();
-
-                let link;
-                const linkEle = routeParams.get('itemLink') ? item.find(routeParams.get('itemLink')) : item;
-                if (routeParams.get('itemLinkAttr')) {
-                    link = linkEle.attr(routeParams.get('itemLinkAttr'));
-                } else {
-                    link = linkEle.is('a') ? linkEle.attr('href') : linkEle.find('a').attr('href');
-                }
-                // 补全绝对链接
-                link = link.trim();
-                if (link && !link.startsWith('http')) {
-                    link = `${new URL(url).origin}${link}`;
+        const routeParams = new URLSearchParams(ctx.req.param('routeParams'));
+        const encoding = routeParams.get('encoding') || 'utf-8';
+        const decoder = new TextDecoder(encoding);
+
+        const $ = load(decoder.decode(response.data));
+        const rssTitle = routeParams.get('title') || $('title').text();
+        const item = routeParams.get('item') || 'html';
+        let items: DataItem[] = $(item)
+            .toArray()
+            .slice(0, 20)
+            .map((item) => {
+                try {
+                    item = $(item);
+
+                    const titleEle = routeParams.get('itemTitle') ? item.find(routeParams.get('itemTitle')) : item;
+                    const title = routeParams.get('itemTitleAttr') ? titleEle.attr(routeParams.get('itemTitleAttr')) : titleEle.text();
+
+                    let link;
+                    const linkEle = routeParams.get('itemLink') ? item.find(routeParams.get('itemLink')) : item;
+                    if (routeParams.get('itemLinkAttr')) {
+                        link = linkEle.attr(routeParams.get('itemLinkAttr'));
+                    } else {
+                        link = linkEle.is('a') ? linkEle.attr('href') : linkEle.find('a').attr('href');
+                    }
+                    // 补全绝对链接
+                    link = link.trim();
+                    if (link && !link.startsWith('http')) {
+                        link = `${new URL(url).origin}${link}`;
+                    }
+
+                    const descEle = routeParams.get('itemDesc') ? item.find(routeParams.get('itemDesc')) : item;
+                    const desc = routeParams.get('itemDescAttr') ? descEle.attr(routeParams.get('itemDescAttr')) : descEle.html();
+
+                    const pubDateEle = routeParams.get('itemPubDate') ? item.find(routeParams.get('itemPubDate')) : item;
+                    const pubDate = routeParams.get('itemPubDateAttr') ? pubDateEle.attr(routeParams.get('itemPubDateAttr')) : pubDateEle.html();
+
+                    return {
+                        title,
+                        link,
+                        description: desc,
+                        pubDate,
+                    };
+                } catch {
+                    return null;
                 }
+            })
+            .filter((i) => !!i);
 
-                const descEle = routeParams.get('itemDesc') ? item.find(routeParams.get('itemDesc')) : item;
-                const desc = routeParams.get('itemDescAttr') ? descEle.attr(routeParams.get('itemDescAttr')) : descEle.html();
-
-                const pubDateEle = routeParams.get('itemPubDate') ? item.find(routeParams.get('itemPubDate')) : item;
-                const pubDate = routeParams.get('itemPubDateAttr') ? pubDateEle.attr(routeParams.get('itemPubDateAttr')) : pubDateEle.html();
-
-                return {
-                    title,
-                    link,
-                    description: desc,
-                    pubDate,
-                };
-            } catch {
-                return null;
-            }
-        })
-        .filter(Boolean);
-
-    return {
-        title: rssTitle,
-        link: url,
-        description: `Proxy ${url}`,
-        item: items,
-    };
-}
+        const itemContentSelector = routeParams.get('itemContent');
+        if (itemContentSelector) {
+            items = await Promise.all(
+                items.map((item) => {
+                    if (!item.link) {
+                        return item;
+                    }
+
+                    return cache.tryGet(item.link, async () => {
+                        const response = await got({
+                            method: 'get',
+                            url: item.link,
+                            responseType: 'arrayBuffer',
+                        });
+                        if (!response || typeof response === 'string') {
+                            return item;
+                        }
+
+                        const $ = load(decoder.decode(response.data));
+                        const content = $(itemContentSelector).html();
+                        if (!content) {
+                            return item;
+                        }
+
+                        item.description = sanitizeHtml(content, { allowedTags: [...sanitizeHtml.defaults.allowedTags, 'img'] });
+
+                        return item;
+                    });
+                })
+            );
+        }
+
+        return {
+            title: rssTitle,
+            link: url,
+            description: `Proxy ${url}`,
+            item: items,
+        };
+    },
+};

package/lib/routes/samd/news.ts

@@ -8,7 +8,7 @@ import ofetch from '@/utils/ofetch';
 const dict = { '434': '行业资讯', '436': '协会动态', '438': '重要通知', '440': '政策法规' };
 
 export const route: Route = {
-    path: 'news/:typeId',
+    path: '/news/:typeId',
     categories: ['government'],
     example: '/samd/news/440',
     parameters: { type: '文章类型ID，见下表' },

package/lib/routes/sciencenet/user.ts

@@ -31,6 +31,7 @@ export const route: Route = {
 
 async function handler(ctx) {
     const id = ctx.req.param('id');
+    const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 50;
 
     const rootUrl = 'https://blog.sciencenet.cn';
     const currentUrl = `${rootUrl}/u/${id}`;
@@ -51,8 +52,9 @@ async function handler(ctx) {
     $ = load(response.data);
 
     let items = $('item')
-        .slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 50)
+        .slice(-limit)
         .toArray()
+        .reverse()
         .map((item) => {
             item = $(item);
 

package/lib/routes/scmp/utils.ts

@@ -1,6 +1,6 @@
 import { load } from 'cheerio';
-import got from '@/utils/got';
 import { parseDate } from '@/utils/parse-date';
+import ofetch from '@/utils/ofetch';
 
 export const renderHTML = (node) => {
     if (!node) {
@@ -60,7 +60,7 @@ export const renderHTML = (node) => {
 };
 
 export const parseItem = async (item) => {
-    const { data: response, url } = await got(item.link);
+    const { _data: response, url } = await ofetch.raw(item.link);
 
     if (new URL(url).hostname !== 'www.scmp.com') {
         // e.g., https://multimedia.scmp.com/

package/lib/routes/scut/gzic/media.ts

@@ -0,0 +1,71 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import ofetch from '@/utils/ofetch';
+
+export const route: Route = {
+    path: '/gzic/media',
+    categories: ['university'],
+    example: '/scut/gzic/media',
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: true,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: '广州国际校区 - 媒体报道',
+    maintainers: ['gdzhht'],
+    handler,
+    description: `:::warning
+由于学校网站对非大陆 IP 的访问存在限制，可能需自行部署。
+:::`,
+};
+
+async function handler() {
+    const url = 'https://www2.scut.edu.cn/gzic/30281/list.htm';
+
+    const { data: response } = await got(url);
+    const $ = load(response);
+
+    const list = $('.right-nr .row .col-lg-4')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const a = item.find('.thr-box a');
+            const pubDate = item.find('.thr-box a span');
+            return {
+                title: item.find('.thr-box a p').text(),
+                link: a.attr('href')?.startsWith('http') ? a.attr('href') : `https://www2.scut.edu.cn${a.attr('href')}`,
+                pubDate: parseDate(pubDate.text()),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                try {
+                    const response = await ofetch(item.link);
+                    const $ = load(response);
+                    item.description = $('div.wp_articlecontent').html();
+                } catch (error) {
+                    if (error.response && error.response.status === 404) {
+                        item.description = '';
+                    } else {
+                        throw error;
+                    }
+                }
+                return item;
+            })
+        )
+    );
+
+    return {
+        title: '华南理工大学广州国际校区 - 媒体报道',
+        link: url,
+        item: items,
+    };
+}

package/lib/routes/scut/gzic/news.ts

@@ -0,0 +1,65 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import ofetch from '@/utils/ofetch';
+
+export const route: Route = {
+    path: '/gzic/news',
+    categories: ['university'],
+    example: '/scut/gzic/news',
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: true,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: '广州国际校区 - 新闻聚焦',
+    maintainers: ['gdzhht'],
+    handler,
+    description: `:::warning
+由于学校网站对非大陆 IP 的访问存在限制，可能需自行部署。
+:::`,
+};
+
+async function handler() {
+    const baseUrl = 'https://www2.scut.edu.cn';
+    const url = 'https://www2.scut.edu.cn/gzic/30279/list.htm';
+
+    const { data: response } = await got(url);
+    const $ = load(response);
+
+    const list = $('.right-nr .row .col-lg-4')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const a = item.find('.li-img a');
+            const pubDate = item.find('.li-img a span');
+            return {
+                title: item.find('.li-img a p').text(),
+                link: a.attr('href')?.startsWith('http') ? a.attr('href') : `${baseUrl}${a.attr('href')}`,
+                pubDate: parseDate(pubDate.text().replaceAll(/年|月/g, '-').replaceAll('日', '')),
+                itunes_item_image: `${baseUrl}${item.find('.li-img img').attr('src')}`,
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const response = await ofetch(item.link);
+                const $ = load(response);
+                item.description = item.link.startsWith('https://mp.weixin.qq.com/') ? $('div.rich_media_content section').html() : $('div.wp_articlecontent').html();
+                return item;
+            })
+        )
+    );
+
+    return {
+        title: '华南理工大学广州国际校区 - 新闻聚焦',
+        link: url,
+        item: items,
+    };
+}

package/lib/routes/scut/gzic/notice.ts

@@ -0,0 +1,88 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import ofetch from '@/utils/ofetch';
+
+const categoryMap = {
+    xsyg: { title: '学术预告', tag: '30284' },
+    jytz: { title: '教研通知', tag: '30307' },
+    hwxx: { title: '海外学习', tag: 'hwxx' },
+    swtz: { title: '事务通知', tag: '30283' },
+};
+
+export const route: Route = {
+    path: '/gzic/notice/:category?',
+    categories: ['university'],
+    example: '/scut/gzic/notice/swtz',
+    parameters: { category: '通知分类，默认为 `swtz`' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: true,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: '广州国际校区 - 通知公告',
+    maintainers: ['gdzhht'],
+    handler,
+    description: `| 学术预告 | 教研通知 | 海外学习 | 事务通知 |
+  | -------- | -------- | -------- | -------- |
+  | xsyg     | jytz     | hwxx     | swtz     |
+
+:::warning
+由于学校网站对非大陆 IP 的访问存在限制，可能需自行部署。
+部分通知详情页可能会被删除（返回 404），或在校园网外无法访问。
+:::`,
+};
+
+async function handler(ctx) {
+    const baseUrl = 'https://www2.scut.edu.cn';
+
+    const categoryName = ctx.req.param('category') || 'swtz';
+    const categoryMeta = categoryMap[categoryName];
+    const url = `${baseUrl}/gzic/${categoryMeta.tag}/list.htm`;
+
+    const { data: response } = await got(url);
+    const $ = load(response);
+
+    const list = $('.right-nr .row .col-lg-4')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const a = item.find('.thr-box a');
+            const pubDate = item.find('.thr-box a span');
+            return {
+                title: item.find('.thr-box a p').text(),
+                link: a.attr('href')?.startsWith('http') ? a.attr('href') : `${baseUrl}${a.attr('href')}`,
+                pubDate: parseDate(pubDate.text()),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                try {
+                    const response = await ofetch(item.link);
+                    const $ = load(response);
+                    item.description = $('div.wp_articlecontent').html();
+                } catch (error) {
+                    if (error.response && error.response.status === 404) {
+                        item.description = '';
+                    } else {
+                        throw error;
+                    }
+                }
+                return item;
+            })
+        )
+    );
+
+    return {
+        title: `华南理工大学广州国际校区 - ${categoryMeta.title}`,
+        link: url,
+        item: items,
+    };
+}